Merge branch 'v3' into PMM-13235-grafana-11

.gitignore

@@ -42,3 +42,5 @@ cli-tests/playwright-report/
 
 api-tests/pmm-api-tests-output.txt
 api-tests/pmm-api-tests-junit-report.xml
+
+packer.log

CONTRIBUTING.md

@@ -44,7 +44,6 @@ This project is built from several repositories:
 * [percona/proxysql_exporter](https://github.com/percona/proxysql_exporter) exports ProxySQL server's metrics
 * [percona/rds_exporter](https://github.com/percona/rds_exporter) exports metrics from RDS
 * [percona/azure_exporter](https://github.com/percona/azure_metrics_exporter) exports metrics from Azure
-* [Percona-Lab/clickhouse_exporter](https://github.com/Percona-Lab/clickhouse_exporter) exports metrics from ClickHouse
 * [percona/percona-toolkit](https://github.com/percona/percona-toolkit) is a collection of advanced command-line tools to perform a variety of MySQL and system tasks that are too difficult or complex to perform manually
 
 
@@ -117,8 +116,8 @@ Since PMM has a lot of components, we will mention only three big parts of it.
 ### PMM Server
 
 * Clone [pmm repository](https://github.com/percona/pmm)
-* Run `make env-up` to start development container. This will be slow on first run, all consequent calls will be order of magnitude faster, because development container will be reused. From time to time it is recommended to perform container rebuild to pull the latest changes, for that run `make env-up-rebuild`.
-* To run pmm-managed with a new changes just run `make env TARGET="run-managed"`, it updates `pmm-managed` running in container.
+* Run `make env-up` to start development container. This will be slow on first run, all subsequent runs will be order of magnitude faster, because development container will be reused. From time to time it is recommended to rebuild the container to pull the latest changes by running `make env-up-rebuild`.
+* To run pmm-managed with your code changes, just run `make run-managed`. It updates `pmm-managed` running in the container.
 
 ### PMM Client
 
@@ -127,11 +126,11 @@ Since PMM has a lot of components, we will mention only three big parts of it.
 * Run `make setup-dev` to connect pmm-agent to PMM Server.
   * This command will register local pmm-agent to PMM Server and generate config file `pmm-agent-dev.yaml`
 * Once it's connected just use `make run` to run pmm-agent.
-* To work correctly pmm-agent needs vmagent and exporters installed on the system.
-  * First option is just install pmm-client using this instrucion https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#install. It will install all exporters as well.
+* To work correctly, pmm-agent needs vmagent and exporters installed on the system.
+  * The first option is to install pmm-client using this instrucion https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#install. It will install all exporters as well.
   * Another option is to do it manually
-    * vmagent and exporters can be installed by building each of them or by downloading the pmm-client tarball from [percona.com](https://www.percona.com/downloads/pmm/) and copying binaries to the exporters_base directory configured in a `pmm-agent-dev.yaml` file.
-    * All paths to exporters binaries are configured in `pmm-agent-dev.yaml`, so they can be changed manually
+    * vmagent and exporters can be installed by building each of them or by downloading the pmm-client tarball from [percona.com](https://www.percona.com/downloads/pmm2/) and copying binaries to the exporters_base directory configured in `pmm-agent-dev.yaml` file.
+    * All paths to exporter binaries are configured in `pmm-agent-dev.yaml`, so they can be changed manually if necessary.
 
 ### Exporters
 
@@ -143,20 +142,24 @@ See [Grafana Dashboards Contribution Guide](https://github.com/percona/grafana-d
 
 ## Tests
 
-In a PMM we have 3 kind of tests.
+In PMM we have 3 kinds of tests:
+
+  - unit tests
+  - API tests
+  - end-to-end, or e2e, tests
 
 ### Unit tests
 
-The first one is a Unit testing, so we have unit tests in each repository mentioned above. each of repositories has it's own instruction how to run unit tests.
+Each repository mentioned above has its own set of unit tests, as well as its own instruction on how to run unit tests.
 
 ### API tests
 
-API tests are included into pmm repository and located in [api-tests directory](https://github.com/percona/pmm/tree/main/api-tests). API tests runs against running PMM Server container.
+API tests are part of the PMM repository and can be found in [api-tests directory](https://github.com/percona/pmm/tree/main/api-tests). API tests run inside of an active PMM Server container.
 
-### End to End (E2E) tests
+### End-to-end (E2E) tests
 
-End to End tests are located in [pmm-qa repository](https://github.com/percona/pmm-qa). They includes UI tests and CLI tests.
-Please see [readme](https://github.com/percona/pmm-qa#readme) for details on how to run these.
+End-to-end tests are located in [pmm-qa repository](https://github.com/percona/pmm-qa). They include UI tests and CLI tests.
+Please refer to [readme](https://github.com/percona/pmm-qa#readme) for details on how to run these.
 
 ## Submitting a Pull Request
 
@@ -165,49 +168,53 @@ Before proceeding with your first pull request, we highly recommend you to read
 - [Tech stack](docs/process/tech_stack.md)
 - [Best practices](docs/process/best_practices.md)
 
-As a PR created you are responsible to:
-* make sure PR is ready (linted, tested and etc)
-* make sure it is reviewed (ask for review, ping reviewers and etc)
-* make sure it is merged
-  * merge when it is reviewed and tested
-  * ask code owners/admins to merge it if merging is blocked for some reason
+Once your PR is created, please do the following:
+* prepare your PR for review
+  * run code syntax checks, or linters
+  * run tests and make sure they all pass
+* pass the review (ask for review, ping reviewers)
+* then merge it
+  * ask code owners or admins to merge it if merging is blocked for some reason
 
 ## Feature Build
 
-PMM is quite complex project, it consists from many different repos descibed above. Feature Build (FB) is a way to get changes all together, build them all together, run tests and get client and server containers.
+PMM is quite a complex project, it consists of many different repos described above. A Feature Build (FB) is a way to put everything together, build all components, run tests and, finally, build client and server containers.
 
 Please see: [How to create a feature build](https://github.com/Percona-Lab/pmm-submodules/blob/v3/README.md#how-to-create-a-feature-build)
 
-### The Goals of the Feature Builds
+### The Goals of Feature Builds
 
-1. Provide an easy way to test/accept functionality for PO/PM and QA
-2. Inform the Developer about Automation Tests results before the code is merged
-3. (Future) Let the Developers add/modify e2e tests when they change functionality
+1. Provide a way to have the functionality tested by the developer and QA (or other PMM team members)
+2. Inform the Developer about Automation Test results before the code is merged
+3. Let the Developers add or modify e2e tests whenever there are functional changes
 
 ### The Rules
 
-1. Start Feature Build for every feature/improvement you are working on.
-2. Start PullRequest to percona-lab/pmm-submodules as DRAFT.
-3. Change the status of Pull Request from Draft to Open ONLY if your changes must be merged to pmm-submodules.
-4. Include a short explanation in the Long Description field of the Feature in PR for feature build and checkboxes to all related Pull Requests. Check other [PRs](https://github.com/Percona-Lab/pmm-submodules/pulls) as examples.
+1. Create a Feature Build for every feature/improvement/bugfix you are working on.
+2. Create a draft Pull Request in https://percona-lab/pmm-submodules.
+3. Change the status of the Pull Request from Draft to Open ONLY if you are contributing code changes to pmm-submodules (very rare).
+4. Provide a short explanation in the Description field of you feature build PR and checkboxes to all related Pull Requests. If you need examples, check out [PRs](https://github.com/Percona-Lab/pmm-submodules/pulls) made by others.
 5. After all related PRs in feature build are merged you should:
    a. either close the PR and delete the branch (this is the default option) or
-   b. merge the PR to pmm-submodules repository (please note, this rarely needs to be merged, for example infrastructure changes do)
+   b. merge the PR to pmm-submodules repository (please note, this rarely needs to be merged, for example infrastructure changes)
 
 ## Code Reviews
 
-There are number of approaches for the code review and ownership: Code Ownership (CODEOWNERS), [github auto review](https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/managing-code-review-assignment-for-your-team), PR owner assign ppl that are better fit for the particular code/job.
+There is a number of approaches we use for the code review and ownership: 
+
+- code ownership, which is enforced via github's CODEOWNERS file
+- github [code review assignment](https://docs.github.com/en/github/setting-up-and-managing-organizations-and-teams/managing-code-review-assignment-for-your-team)
+- finally, a PR owner can manually assign reviewers (usually one or more PMM team members).
 
-For more efficient review process we use a mixed approach:
-* repos that have CODEOWNERS
+To make the review process effective, we use a mixed approach:
+* for repos that have CODEOWNERS
   * github will assign reviewers automatically
-* repos that don't have CODEOWNERS
+* for repos that don't have CODEOWNERS
   * add reviewers as follows:
       * add `pmm-review-fe` for UI/UX reviews
       * add `pmm-review-exporters` for exporter reviews [see PMM Client](#PMM-Client)
       * add `pmm-review-be` for backend reviews
-* if you know exactly who should review your code
-  * add them to the review
+* if you know exactly who should review your code, add them to the review
 
 
 | Team                 | Description                                                    | Members |

Dockerfile

@@ -1,6 +1,6 @@
 # This Dockerfile is used only for API tests.
 
-FROM golang:1.22
+FROM golang:1.23
 
 RUN export GOPATH=$(go env GOPATH) && \
     mkdir -p $GOPATH/src/github.com/percona/pmm

agent/agents/postgres/pgstatstatements/pgstatstatements.go

@@ -287,7 +287,7 @@ func (m *PGStatStatementsQAN) getNewBuckets(ctx context.Context, periodStart tim
 		return nil, err
 	}
 
-	buckets := makeBuckets(current, prev, m.disableCommentsParsing, m.l)
+	buckets := m.makeBuckets(current, prev)
 	startS := uint32(periodStart.Unix())
 	m.l.Debugf("Made %d buckets out of %d stat statements in %s+%d interval.",
 		len(buckets), len(current), periodStart.Format("15:04:05"), periodLengthSecs)
@@ -312,8 +312,9 @@ func (m *PGStatStatementsQAN) getNewBuckets(ctx context.Context, periodStart tim
 
 // makeBuckets uses current state of pg_stat_statements table and accumulated previous state
 // to make metrics buckets. It's a pure function for easier testing.
-func makeBuckets(current, prev statementsMap, disableCommentsParsing bool, l *logrus.Entry) []*agentv1.MetricsBucket {
+func (m *PGStatStatementsQAN) makeBuckets(current, prev statementsMap) []*agentv1.MetricsBucket {
 	res := make([]*agentv1.MetricsBucket, 0, len(current))
+	l := m.l
 
 	for queryID, currentPSS := range current {
 		prevPSS := prev[queryID]
@@ -339,10 +340,10 @@ func makeBuckets(current, prev statementsMap, disableCommentsParsing bool, l *lo
 		}
 
 		if len(currentPSS.Tables) == 0 {
-			currentPSS.Tables = extractTables(currentPSS.Query, l)
+			currentPSS.Tables = extractTables(currentPSS.Query, m.maxQueryLength, l)
 		}
 
-		if !disableCommentsParsing {
+		if !m.disableCommentsParsing {
 			comments, err := queryparser.PostgreSQLComments(currentPSS.Query)
 			if err != nil {
 				l.Errorf("failed to parse comments for query: %s", currentPSS.Query)

agent/agents/postgres/pgstatstatements/utils.go

@@ -72,9 +72,9 @@ func queryUsernames(q *reform.Querier) map[int64]string {
 	return res
 }
 
-func extractTables(query string, l *logrus.Entry) []string {
+func extractTables(query string, maxQueryLength int32, l *logrus.Entry) []string {
 	start := time.Now()
-	t, _ := truncate.Query(query, truncate.GetDefaultMaxQueryLength())
+	t, _ := truncate.Query(query, maxQueryLength)
 	tables, err := parser.ExtractTables(query)
 	if err != nil {
 		// log full query and error stack on debug level or more

agent/agents/process/process.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
@@ -53,12 +54,14 @@ const (
 // implements its own logic, and then switches to then next state via "go toXXX()". "go" statement is used
 // only to avoid stack overflow; there are no extra goroutines for states.
 type Process struct {
-	params  *Params
-	l       *logrus.Entry
-	pl      *processLogger
-	changes chan inventoryv1.AgentStatus
-	backoff *backoff.Backoff
-	ctxDone chan struct{}
+	params      *Params
+	l           *logrus.Entry
+	pl          *processLogger
+	changes     chan inventoryv1.AgentStatus
+	backoff     *backoff.Backoff
+	ctxDone     chan struct{}
+	err         error
+	initialized chan bool
 
 	// recreated on each restart
 	cmd     *exec.Cmd
@@ -88,15 +91,26 @@ func (p *Params) String() string {
 // New creates new process.
 func New(params *Params, redactWords []string, l *logrus.Entry) *Process {
 	return &Process{
-		params:  params,
-		l:       l,
-		pl:      newProcessLogger(l, keepLogLines, redactWords),
-		changes: make(chan inventoryv1.AgentStatus, 10),
-		backoff: backoff.New(backoffMinDelay, backoffMaxDelay),
-		ctxDone: make(chan struct{}),
+		params:      params,
+		l:           l,
+		pl:          newProcessLogger(l, keepLogLines, redactWords),
+		changes:     make(chan inventoryv1.AgentStatus, 10),
+		backoff:     backoff.New(backoffMinDelay, backoffMaxDelay),
+		ctxDone:     make(chan struct{}),
+		initialized: make(chan bool, 1),
 	}
 }
 
+// IsInitialized returns a chan of bool. True can be received if the process is initialized.
+func (p *Process) IsInitialized() <-chan bool {
+	return p.initialized
+}
+
+// GetError returns the error thrown when initializing the process.
+func (p *Process) GetError() error {
+	return p.err
+}
+
 // Run starts process and runs until ctx is canceled.
 func (p *Process) Run(ctx context.Context) {
 	go p.toStarting()
@@ -107,7 +121,7 @@ func (p *Process) Run(ctx context.Context) {
 }
 
 // STARTING -> RUNNING.
-// STARTING -> WAITING.
+// STARTING -> FAILING.
 func (p *Process) toStarting() {
 	p.l.Tracef("Process: starting.")
 	p.changes <- inventoryv1.AgentStatus_AGENT_STATUS_STARTING
@@ -128,7 +142,7 @@ func (p *Process) toStarting() {
 
 	if err := p.cmd.Start(); err != nil {
 		p.l.Warnf("Process: failed to start: %s.", err)
-		go p.toWaiting()
+		go p.toFailing(err)
 		return
 	}
 
@@ -142,10 +156,11 @@ func (p *Process) toStarting() {
 	defer t.Stop()
 	select {
 	case <-t.C:
+		p.initialized <- true
 		go p.toRunning()
 	case <-p.cmdDone:
 		p.l.Warnf("Process: exited early: %s.", p.cmd.ProcessState)
-		go p.toWaiting()
+		go p.toFailing(errors.New("exited early"))
 	}
 }
 
@@ -192,6 +207,16 @@ func (p *Process) toWaiting() {
 	}
 }
 
+// FAILING -> DONE.
+func (p *Process) toFailing(err error) {
+	p.l.Tracef("Process: failing")
+	p.changes <- inventoryv1.AgentStatus_AGENT_STATUS_INITIALIZATION_ERROR
+	p.l.Infof("Process: exited: %s.", p.cmd.ProcessState)
+	go p.toDone()
+	p.err = err
+	p.initialized <- false
+}
+
 // STOPPING -> DONE.
 func (p *Process) toStopping() {
 	p.l.Tracef("Process: stopping (sending SIGTERM)...")

agent/agents/process/process_test.go

@@ -80,35 +80,22 @@ func TestProcess(t *testing.T) {
 	})
 
 	t.Run("FailedToStart", func(t *testing.T) {
-		ctx, cancel, l := setup(t)
+		ctx, _, l := setup(t)
 		p := New(&Params{Path: "no_such_command"}, nil, l)
 		go p.Run(ctx)
 
-		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_WAITING, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_WAITING)
-		cancel()
-		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_DONE, inventoryv1.AgentStatus_AGENT_STATUS_UNSPECIFIED)
+		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_INITIALIZATION_ERROR,
+			inventoryv1.AgentStatus_AGENT_STATUS_DONE, inventoryv1.AgentStatus_AGENT_STATUS_UNSPECIFIED)
 	})
 
 	t.Run("ExitedEarly", func(t *testing.T) {
 		sleep := strconv.FormatFloat(runningT.Seconds()-0.5, 'f', -1, 64)
-		ctx, cancel, l := setup(t)
-		p := New(&Params{Path: "sleep", Args: []string{sleep}}, nil, l)
-		go p.Run(ctx)
-
-		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_WAITING, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_WAITING)
-		cancel()
-		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_DONE, inventoryv1.AgentStatus_AGENT_STATUS_UNSPECIFIED)
-	})
-
-	t.Run("CancelStarting", func(t *testing.T) {
-		sleep := strconv.FormatFloat(runningT.Seconds()-0.5, 'f', -1, 64)
-		ctx, cancel, l := setup(t)
+		ctx, _, l := setup(t)
 		p := New(&Params{Path: "sleep", Args: []string{sleep}}, nil, l)
 		go p.Run(ctx)
 
-		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_WAITING, inventoryv1.AgentStatus_AGENT_STATUS_STARTING)
-		cancel()
-		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_WAITING, inventoryv1.AgentStatus_AGENT_STATUS_DONE, inventoryv1.AgentStatus_AGENT_STATUS_UNSPECIFIED)
+		assertStates(t, p, inventoryv1.AgentStatus_AGENT_STATUS_STARTING, inventoryv1.AgentStatus_AGENT_STATUS_INITIALIZATION_ERROR,
+			inventoryv1.AgentStatus_AGENT_STATUS_DONE, inventoryv1.AgentStatus_AGENT_STATUS_UNSPECIFIED)
 	})
 
 	t.Run("Exited", func(t *testing.T) {