Merge branch 'v3-api-breaking-changes' into PMM-12913-migrate-api-end…

…points-to-restful

.github/workflows/agent.yml

@@ -94,7 +94,7 @@ jobs:
       - name: Build and install
         run: make install
 
-      - name: Docker-compose
+      - name: Launch the containers
         env:
           ENV_UP_FLAGS: "--detach"
         run: make env-up

.github/workflows/dependabot.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: Dependabot metadata
         id: metadata
-        uses: dependabot/fetch-metadata@v1
+        uses: dependabot/fetch-metadata@v2
         with:
           github-token: "${{ secrets.GITHUB_TOKEN }}"
 

.github/workflows/release-doc.yml

@@ -25,7 +25,7 @@ jobs:
           grep -rl '!!! note alert alert-primary' ${{ github.workspace }}-CHANGELOG.txt | xargs --no-run-if-empty sed -i 's/\!\!\! note alert alert-primary "\(.*\)"/\> \:memo\: **\1**/g'
           grep -rl '!!! note alert alert-primary' ${{ github.workspace }}-CHANGELOG.txt | xargs --no-run-if-empty sed -i 's/\!\!\! note alert alert-primary/\> \:memo\: **Note**/g'
       - name: Create Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           body_path: ${{ github.workspace }}-CHANGELOG.txt
           draft: true

admin/commands/base/setup.go

@@ -88,8 +88,13 @@ func SetupClients(ctx context.Context, globalFlags *flags.GlobalFlags) {
 	// use JSON APIs over HTTP/1.1
 	transport := httptransport.New(globalFlags.ServerURL.Host, globalFlags.ServerURL.Path, []string{globalFlags.ServerURL.Scheme})
 	if u := globalFlags.ServerURL.User; u != nil {
+		user := u.Username()
 		password, _ := u.Password()
-		transport.DefaultAuthentication = httptransport.BasicAuth(u.Username(), password)
+		if user == "service_token" || user == "api_key" {
+			transport.DefaultAuthentication = httptransport.BearerToken(password)
+		} else {
+			transport.DefaultAuthentication = httptransport.BasicAuth(user, password)
+		}
 	}
 	transport.SetLogger(logrus.WithField("component", "server-transport"))
 	transport.SetDebug(globalFlags.EnableDebug || globalFlags.EnableTrace)

admin/commands/management/unregister.go

@@ -15,14 +15,16 @@
 package management
 
 import (
-	"github.com/AlekSi/pointer"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 
 	"github.com/percona/pmm/admin/agentlocal"
 	"github.com/percona/pmm/admin/commands"
 	"github.com/percona/pmm/admin/helpers"
 	"github.com/percona/pmm/api/inventory/v1/json/client"
 	nodes "github.com/percona/pmm/api/inventory/v1/json/client/nodes_service"
+	managementClient "github.com/percona/pmm/api/management/v1/json/client"
+	mservice "github.com/percona/pmm/api/management/v1/json/client/management_service"
 )
 
 type unregisterResult struct {
@@ -77,17 +79,23 @@ func (cmd *UnregisterCommand) RunCmd() (commands.Result, error) {
 		}
 	}
 
-	params := &nodes.RemoveNodeParams{
-		NodeID:  nodeID,
-		Force:   pointer.ToBool(cmd.Force),
+	params := &mservice.UnregisterNodeParams{
+		Body: mservice.UnregisterNodeBody{
+			NodeID: nodeID,
+			Force:  cmd.Force,
+		},
 		Context: commands.Ctx,
 	}
 
-	_, err = client.Default.NodesService.RemoveNode(params)
+	res, err := managementClient.Default.ManagementService.UnregisterNode(params)
 	if err != nil {
 		return nil, err
 	}
 
+	if res.Payload.Warning != "" {
+		logrus.Warning(res.Payload.Warning)
+	}
+
 	return &unregisterResult{
 		NodeID:   nodeID,
 		NodeName: nodeName,

agent/Makefile

@@ -110,10 +110,10 @@ env-up:                         ## Start development environment
 	mkdir -p testdata/mysql/slowlogs
 	chmod -R 0777 testdata
 
-	docker-compose up $(ENV_UP_FLAGS)
+	docker compose up $(ENV_UP_FLAGS)
 
 env-down:                       ## Stop development environment
-	docker-compose down --volumes --remove-orphans
+	docker compose down --volumes --remove-orphans
 
 setup-dev: install              ## Run pmm-agent setup in development environment
 	pmm-agent setup $(RUN_FLAGS) --server-insecure-tls --server-address=127.0.0.1:${PMM_DEV_SERVER_PORT} --server-username=admin --server-password=admin --paths-exporters_base=$(GOPATH)/bin --force
@@ -128,13 +128,13 @@ env-psql:                       ## Run psql client
 	docker exec -ti pmm-agent_postgres env PGPASSWORD=pmm-agent-password psql --username=pmm-agent
 
 env-sysbench-prepare:
-	docker-compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
+	docker compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
 		--db-driver=pgsql --pgsql-host=postgres --pgsql-user=pmm-agent --pgsql-password=pmm-agent-password --pgsql-db=pmm-agent \
 		--threads=1 --time=0 --report-interval=10 \
 		--tables=1 --scale=10  --use_fk=0 --enable_purge=yes prepare
 
 env-sysbench-run:
-	docker-compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
+	docker compose exec --workdir=/sysbench/sysbench-tpcc sysbench ./tpcc.lua \
 		--db-driver=pgsql --pgsql-host=postgres --pgsql-user=pmm-agent --pgsql-password=pmm-agent-password --pgsql-db=pmm-agent \
 		--threads=4 --time=0 --rate=10 --report-interval=10 --percentile=99 \
 		--tables=1 --scale=10  --use_fk=0 --enable_purge=yes run

agent/client/basic_auth.go

@@ -17,6 +17,7 @@ package client
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
 
 	"google.golang.org/grpc/credentials"
 )
@@ -31,7 +32,7 @@ func (b *basicAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[
 	auth := b.username + ":" + b.password
 	enc := base64.StdEncoding.EncodeToString([]byte(auth))
 	return map[string]string{
-		"authorization": "Basic " + enc,
+		"Authorization": fmt.Sprintf("Basic %s", enc),
 	}, nil
 }
 

agent/client/client.go

@@ -748,10 +748,16 @@ func dial(dialCtx context.Context, cfg *config.Config, l *logrus.Entry) (*dialRe
 	}
 
 	if cfg.Server.Username != "" {
-		opts = append(opts, grpc.WithPerRPCCredentials(&basicAuth{
-			username: cfg.Server.Username,
-			password: cfg.Server.Password,
-		}))
+		if cfg.Server.Username == "service_token" || cfg.Server.Username == "api_key" {
+			opts = append(opts, grpc.WithPerRPCCredentials(&tokenAuth{
+				token: cfg.Server.Password,
+			}))
+		} else {
+			opts = append(opts, grpc.WithPerRPCCredentials(&basicAuth{
+				username: cfg.Server.Username,
+				password: cfg.Server.Password,
+			}))
+		}
 	}
 
 	l.Infof("Connecting to %s ...", cfg.Server.FilteredURL())

agent/client/token_auth.go

@@ -0,0 +1,43 @@
+// Copyright (C) 2023 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"google.golang.org/grpc/credentials"
+)
+
+type tokenAuth struct {
+	token string
+}
+
+// GetRequestMetadata implements credentials.PerRPCCredentials interface.
+func (t *tokenAuth) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { //nolint:revive
+	return map[string]string{
+		"Authorization": fmt.Sprintf("Bearer %s", t.token),
+	}, nil
+}
+
+// RequireTransportSecurity implements credentials.PerRPCCredentials interface.
+func (*tokenAuth) RequireTransportSecurity() bool {
+	return false
+}
+
+// check interfaces.
+var (
+	_ credentials.PerRPCCredentials = (*tokenAuth)(nil)
+)

agent/commands/clients.go

@@ -104,8 +104,13 @@ func setServerTransport(u *url.URL, insecureTLS bool, l *logrus.Entry) {
 	// use JSON APIs over HTTP/1.1
 	transport := httptransport.New(u.Host, u.Path, []string{u.Scheme})
 	if u.User != nil {
+		user := u.User.Username()
 		password, _ := u.User.Password()
-		transport.DefaultAuthentication = httptransport.BasicAuth(u.User.Username(), password)
+		if user == "service_token" || user == "api_key" {
+			transport.DefaultAuthentication = httptransport.BearerToken(password)
+		} else {
+			transport.DefaultAuthentication = httptransport.BasicAuth(user, password)
+		}
 	}
 	transport.SetLogger(l)
 	transport.SetDebug(l.Logger.GetLevel() >= logrus.DebugLevel)

agent/commands/setup.go

@@ -160,10 +160,10 @@ func register(cfg *config.Config, l *logrus.Entry) {
 	}
 	cfg.ID = agentID
 	if token != "" {
-		cfg.Server.Username = "api_key"
+		cfg.Server.Username = "service_token"
 		cfg.Server.Password = token
 	} else {
-		l.Info("PMM Server responded with an empty api key token. Consider upgrading PMM Server to the latest version.")
+		l.Info("PMM Server responded with an empty service token. Consider upgrading PMM Server to the latest version.")
 	}
 	fmt.Printf("Registered.\n")
 }

agent/config/config.go

@@ -18,7 +18,6 @@ package config
 import (
 	"fmt"
 	"io/fs"
-	"log"
 	"net"
 	"net/url"
 	"os"
@@ -414,10 +413,11 @@ func Application(cfg *Config) (*kingpin.Application, *string) {
 	}).Bool()
 
 	app.Flag("version", "Show application version").Short('v').Action(func(*kingpin.ParseContext) error {
+		// We use fmt instead of log package to provide proper output for --json flag.
 		if *jsonF {
-			log.Println(version.FullInfoJSON())
+			fmt.Println(version.FullInfoJSON()) //nolint:forbidigo
 		} else {
-			log.Println(version.FullInfo())
+			fmt.Println(version.FullInfo()) //nolint:forbidigo
 		}
 		os.Exit(0)
 

agent/serviceinfobroker/service_info_broker.go

@@ -237,7 +237,7 @@ func (sib *ServiceInfoBroker) getPostgreSQLInfo(ctx context.Context, dsn string,
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
 		res.Error = err.Error()
 	}
-	res.PgsmVersion = pgsmVersion
+	res.PgsmVersion = &pgsmVersion
 
 	return &res
 }

agent/serviceinfobroker/service_info_broker_test.go

@@ -260,7 +260,7 @@ func TestServiceInfoBroker(t *testing.T) {
 		}, 0)
 		require.NotNil(t, resp)
 		assert.Equal(t, []string{"postgres", "pmm-agent"}, resp.DatabaseList)
-		assert.Equal(t, "", resp.PgsmVersion)
+		assert.Equal(t, "", *resp.PgsmVersion)
 	})
 
 	t.Run("MongoDBWithSSL", func(t *testing.T) {

api-tests/backup/backups_test.go

@@ -40,7 +40,7 @@ func TestScheduleBackup(t *testing.T) {
 			NodeName: nodeName,
 			NodeType: pointer.ToString(mservice.RegisterNodeBodyNodeTypeNODETYPEGENERICNODE),
 		})
-		defer pmmapitests.RemoveNodes(t, nodeID)
+		defer pmmapitests.UnregisterNodes(t, nodeID)
 		defer management.RemovePMMAgentWithSubAgents(t, pmmAgentID)
 		mongo1Name := pmmapitests.TestString(t, "mongo")
 		mongo2Name := pmmapitests.TestString(t, "mongo")

api-tests/helpers.go

@@ -17,8 +17,10 @@ package apitests
 
 import (
 	"context"
+	"crypto/rand"
 	"fmt"
-	"math/rand"
+	"math"
+	"math/big"
 	"reflect"
 	"testing"
 
@@ -31,6 +33,8 @@ import (
 	agents "github.com/percona/pmm/api/inventory/v1/json/client/agents_service"
 	nodes "github.com/percona/pmm/api/inventory/v1/json/client/nodes_service"
 	services "github.com/percona/pmm/api/inventory/v1/json/client/services_service"
+	managementClient "github.com/percona/pmm/api/management/v1/json/client"
+	mservice "github.com/percona/pmm/api/management/v1/json/client/management_service"
 )
 
 // ErrorResponse represents the response structure for error scenarios.
@@ -50,7 +54,10 @@ type TestingT interface {
 func TestString(t TestingT, name string) string {
 	t.Helper()
 
-	n := rand.Int() //nolint:gosec
+	// Without proper seed parallel tests can generate same "random" number.
+	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt32))
+	require.NoError(t, err)
+
 	return fmt.Sprintf("pmm-api-tests/%s/%s/%s/%d", Hostname, t.Name(), name, n)
 }
 
@@ -131,6 +138,26 @@ func (tt *expectedFailureTestingT) Check() {
 	tt.t.Fatalf("%s expected to fail, but didn't: %s", tt.Name(), tt.link)
 }
 
+// UnregisterNodes unregister specified nodes.
+func UnregisterNodes(t TestingT, nodeIDs ...string) {
+	t.Helper()
+
+	for _, nodeID := range nodeIDs {
+		params := &mservice.UnregisterNodeParams{
+			Body: mservice.UnregisterNodeBody{
+				NodeID: nodeID,
+			},
+			Context: context.Background(),
+		}
+
+		res, err := managementClient.Default.ManagementService.UnregisterNode(params)
+		require.NoError(t, err)
+		assert.NotNil(t, res)
+		assert.NotNil(t, res.Payload)
+		assert.Empty(t, res.Payload.Warning)
+	}
+}
+
 // RemoveNodes removes specified nodes.
 func RemoveNodes(t TestingT, nodeIDs ...string) {
 	t.Helper()

api-tests/inventory/nodes_test.go

@@ -33,7 +33,6 @@ import (
 )
 
 func TestNodes(t *testing.T) {
-	t.Parallel()
 	t.Run("List", func(t *testing.T) {
 		t.Parallel()
 
@@ -103,7 +102,6 @@ func TestNodes(t *testing.T) {
 }
 
 func TestGetNode(t *testing.T) {
-	t.Parallel()
 	t.Run("Basic", func(t *testing.T) {
 		t.Parallel()
 
@@ -157,7 +155,6 @@ func TestGetNode(t *testing.T) {
 }
 
 func TestGenericNode(t *testing.T) {
-	t.Parallel()
 	t.Run("Basic", func(t *testing.T) {
 		t.Parallel()
 
@@ -222,7 +219,6 @@ func TestGenericNode(t *testing.T) {
 }
 
 func TestContainerNode(t *testing.T) {
-	t.Parallel()
 	t.Run("Basic", func(t *testing.T) {
 		t.Parallel()
 
@@ -292,7 +288,6 @@ func TestContainerNode(t *testing.T) {
 }
 
 func TestRemoteNode(t *testing.T) {
-	t.Parallel()
 	t.Run("Basic", func(t *testing.T) {
 		t.Parallel()
 
@@ -361,7 +356,6 @@ func TestRemoteNode(t *testing.T) {
 }
 
 func TestRemoveNode(t *testing.T) {
-	t.Parallel()
 	t.Run("Basic", func(t *testing.T) {
 		t.Parallel()