PMM-12913 Migrate api endpoints to REST (#2880)

* PMM-12913 Add a resource name to endpoints ending with /v1

* PMM-12913 fix the linter error (goconst)

* PMM-12913 migrate /v1/version

* PMM-12913 migrate /v1/readyz

* PMM-12913 all endpoints begin with `/v1`

* PMM-12913 make gen

* PMM-12913 migrate /v1/settings/Get

* PMM-12913 fix unit test fo /v1/server/settings

* PMM-12913 migrate /v1/settings/Change

* PMM-12913 migrate /v1/AWSInstanceCheck

* PMM-12913 fix alerting api tests

* PMM-12913 fix ManagementService being disabled

* PMM-12913 fix backup API tests

* PMM-12913 temporarily allow /v1/version

* PMM-12913 update admins' readme

* PMM-12913 fix the Agent connect path in auth

* PMM-12913 fix wrong grpc paths

* PMM-12913 fix the port in readme

* PMM-12913 fix some grpc paths

* PMM-12913 add check-agent-setup script

* PMM-12913 migrate /v1/server/leaderHealthCheck endpoint

* PMM-12913 migrate /v1/settings/Get

* PMM-12913 fix the api test

* PMM-12913 migrate /v1/updates/Start

* PMM-12913 fix actions

* PMM-12913 migrate /logs.zip

* PMM-12913  fix auth_server_test

* PMM-12913 migrate /v1/updates/Status

* PMM-12913 migrate /v1/users

* PMM-12913 migrate /v1/inventory/Agents

* PMM-12913 fix logs_test

* PMM-12913 use an interface for commonAgentParams

* PMM-12913 re-add `/agent_id/`

* PMM-12913 migrate /v1/inventory/Agents/Get

* PMM-12913 migrate /v1/inventory/Agents/Remove

* PMM-12913 migrate /v1/inventory/Agents/GetLogs

* PMM-12913 migrate /v1/inventory/Nodes

* PMM-12913 migrate /v1/inventory/Services

* PMM-12913 fix tests for /v1/inventory/services

* PMM-12913 migrate /v1/management/Service

* PMM-12913 migrate /v1/actions

* PMM-12899 migrate /v1/alerting endpoints

* PMM-12899 migrate /v1/advisors emdpoints

* PMM-12899 rename BackupsService to BackupService

* PMM-12899 migrate /v1/backup/Artifacts

* PMM-12899 migrate /v1/backup/Backups

* PMM-12913 refactor /v1/backups/locations

* PMM-12913 refactor /v1/backups/restores

* PMM-12913 refactor /v1/dumps

* PMM-12913 move dump and role API to v1beta1 folder

* PMM-12913 refactor /v1/role, rename tp /v1/accesscontrol

* PMM-12913 refactor /v1/management/nodes {Get,LIst}

* PMM-12913 refactor /v1/management/services/List

* PMM-12913 refactor /v1/management/services/azure

* PMM-12913 refactor /v1/management/agent/List

* PMM-12913 migrate /v1/platform

* PMM-12913 migrate /v1/qan

* PMM-12913 refactor the primitive types in alerting.proto

* PMM-12913 fix a few linter errors

* PMM-12913 do not show CollectorService in swagger

* PMM-12913 split /backup/GetLogs up to two endpoints

* PMM-12913 fix restore tests

* PMM-12913 update migration examples

* PMM-12913 fix /v1/qan/metrics:getFilters being unavailable

* PMM-12913 fix the linter error

* PMM-12913 fix /v1/qan/metrics:getFilters test

* PMM-12913 update migration examples

* PMM-12913 fix an iteration bug

* PMM-12913 fix an issue with QAN failing to persist data

* PMM-12913 remove debug output

* PMM-12913 auth_server fixes for QAN

* PMM-12913 fix agent API tests

* PMM-12913 fix the QAN URL routing issue

* PMM-12913 remove the unnecessary PMM_NICER_API

* PMM-12913 add the routing error handler

* PMM-12913 minor syntax fixes

* PMM-12913 remove PMM_LESS_LOG_NOISE envvar

* PMM-12913 remove ID normalizers

* PMM-12913 update the migration doc

* PMM-12913 better error handling

* PMM-12913 fix /v1/backups/artifacts/{artifact_id} delete endpoint

* PMM-12913 simplify setup.py

* PMM-12913 fix a linter error

* PMM-12913 merge ArtifactService with BackupService

* Revert "PMM-12913 better error handling"

This reverts commit add2d3f.

* PMM-12913: regenerate code, fix Makefile.include

* PMM-12913: fix node_test

* PMM-12913: fix UnregisterNode method name

* PMM-12857 fix API tests

* PMM-12857 fix agent tests

* PMM-12857 fix auth_test and clean up readyz_test

* PMM-12857 fix settings_test

* PMM-12857 fix auth_test

* PMM-13054 fix the backup test

* PMM-13054 fix advisor tests

* PMM-13054 fix auth_server tests again

* PMM-13054 remove redundant functionality

* PMM-13054 fix actions tests

* PMM-13054 minor cleanup

* PMM-13054 disable platform tests until the env is ready

* PMM-13054 fix advisors_test

* PMM-13054 try better error msg to debug failures

* PMM-13054 fix alerting_test

* Revert "PMM-13054 try better error msg to debug failures"

This reverts commit 862ec3b.

.devcontainer/setup.py

@@ -47,16 +47,7 @@ def install_go():
 
     go_version = str(subprocess.check_output("gimme -r " + GO_VERSION, shell=True).strip().decode())
 
-    if GO_VERSION == "tip":
-        run_commands([
-            "mkdir $HOME/git_source",
-            "wget https://github.com/git/git/archive/refs/tags/v2.34.4.tar.gz -O $HOME/git.tar.gz",
-            "tar -xzf $HOME/git.tar.gz -C $HOME/git_source --strip-components 1",
-            "cd $HOME/git_source && make configure && ./configure --prefix=/usr && make all && make install",
-        ])
-        gimme_go_dir = "go"
-    else:
-        gimme_go_dir = "go{go_version}.linux.amd64".format(go_version=go_version)
+    gimme_go_dir = "go{go_version}.linux.amd64".format(go_version=go_version)
 
     run_commands([
         "gimme " + go_version,

.github/workflows/main.yml

@@ -81,22 +81,23 @@ jobs:
 
       - name: Run API linter
         env:
-          COMMAND: 'bin/buf lint -v api'
-          REDIRECT: "| bin/reviewdog -f=buf -reporter=github-pr-review -fail-on-error=true"
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
-          if out=$( ${{ env.COMMAND }} ); exit_code=$?; [ $exit_code -ne 0 ]; then
-            ## buf uses exit code 100 for linter warnings
-            if [ $exit_code != 100 ] || ${{ github.event.pull_request == null }}; then
-              echo "$out"
-              exit $exit_code
-            else
-              echo "$out" ${{ env.REDIRECT }}
-            fi
-          else
+          if out=$(bin/buf lint -v api); code="$?"; test "$code" -eq 0; then
             echo "$out"
+            exit 0
           fi
 
+          echo "API linter exited with code: $code"
+          echo "$out"
+          ## buf uses exit code 100 for linter warnings
+          if [ "$code" -ne 100 ] || ${{ github.event.pull_request == null }}; then
+            exit $code
+          fi
+
+          # One may need to suppress passing to reviewdog because of https://github.com/reviewdog/reviewdog/issues/1696
+          echo "$out" | bin/reviewdog -f=buf -reporter=github-pr-review -fail-on-error=true
+
       - name: Run code linters
         uses: reviewdog/action-golangci-lint@v2
         with:
@@ -110,20 +111,20 @@ jobs:
 
       - name: Run go-consistent
         env:
-          COMMAND: 'bin/go-consistent -pedantic -exclude "tests" ./...'
           REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.ROBOT_TOKEN || secrets.GITHUB_TOKEN }}
         run: |
-          if out=$( ${{ env.COMMAND }} ); exit_code=$?; [ $exit_code -ne 0 ]; then
-            if [ $exit_code -gt 1 ] || ${{ github.event.pull_request == null }}; then
-              echo "$out"
-              exit $exit_code
-            else
-              echo "$out" | bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error=true
-            fi
-          else
+          if out=$(bin/go-consistent -pedantic -exclude "tests" ./...); exit_code=$?; [ $exit_code -eq 0 ]; then
             echo "$out"
+            exit 0
           fi
 
+          if [ $exit_code -gt 1 ] || ${{ github.event.pull_request == null }}; then
+            echo "$out"
+            exit $exit_code
+          fi
+
+          echo "$out" | bin/reviewdog -f=go-consistent -reporter=github-pr-review -fail-on-error=true
+
       - name: Test common API
         run: make test-common
 

.github/workflows/managed.yml

@@ -43,6 +43,7 @@ jobs:
       - name: Launch PMM Server (see docker-compose.yml)
         run: |
           make env-compose-up # the container workdir is /root/go/src/github.com/percona/pmm
+          docker exec -t pmm-server id -Z || :
           docker logs pmm-server
 
       - name: Mark the root directory of pmm as safe
@@ -123,7 +124,8 @@ jobs:
           docker exec pmm-server go env | sort
           docker exec pmm-server supervisorctl status || true
           services=$(docker exec pmm-server supervisorctl status | awk '{print $1}')
-          while IFS= read -r service; do
+          services=(`echo ${services}`)
+          for service in "${services[@]}"; do
             echo "Logs for $service:"
             docker exec pmm-server supervisorctl tail $service
-          done <<< "$services"
+          done

.golangci.yml

@@ -149,6 +149,7 @@ issues:
         - forcetypeassert  # for tests' brevity sake
         - funlen           # tests may be long
         - gocognit         # triggered by subtests
+        - goconst          # not critical for tests
         - gomnd            # tests are full of magic numbers
         - ireturn          # we have exceptions, so need to silence them in tests
         - lll              # tests often require long lines

Makefile.include

@@ -48,7 +48,22 @@ gen-api:            ## Generate PMM API
 
 	bin/buf generate -v api
 
-	for API in api/agentlocal/v1 api/server/v1 api/inventory/v1 api/management/v1 api/actions/v1 api/advisors/v1 api/alerting/v1 api/backup/v1 api/dump/v1 api/management/v1/azure api/role/v1 api/qan/v1 api/management/v1/agent api/management/v1/node api/management/v1/service api/platform/v1 api/user/v1; do \
+	SPECS="\
+		api/agentlocal/v1 \
+		api/server/v1 \
+		api/user/v1 \
+		api/inventory/v1 \
+		api/management/v1 \
+		api/management/v1/service \
+		api/actions/v1 \
+		api/advisors/v1 \
+		api/alerting/v1 \
+		api/backup/v1 \
+		api/dump/v1beta1 \
+		api/accesscontrol/v1beta1 \
+		api/qan/v1 \
+		api/platform/v1"; \
+	for API in $$SPECS; do \
 		set -x ; \
 		bin/swagger mixin $$API/json/header.json $$API/*.swagger.json --output=$$API/json/$$(basename $$API).json --keep-spec-order; \
 		bin/swagger flatten --with-flatten=expand --with-flatten=remove-unused $$API/json/$$(basename $$API).json --output=$$API/json/$$(basename $$API).json ; \
@@ -73,15 +88,14 @@ gen-api:            ## Generate PMM API
 		api/swagger/header.json \
 		api/server/v1/json/v1.json \
 		api/user/v1/json/v1.json \
-    api/inventory/v1/json/v1.json \
+		api/inventory/v1/json/v1.json \
 		api/management/v1/json/v1.json \
 		api/actions/v1/json/v1.json \
 		api/alerting/v1/json/v1.json \
 		api/advisors/v1/json/v1.json \
 		api/backup/v1/json/v1.json \
-    api/dump/v1/json/v1.json \
-    api/role/v1/json/v1.json \
-    api/qan/v1/json/v1.json
+		api/qan/v1/json/v1.json \
+		api/platform/v1/json/v1.json
 	bin/swagger validate api/swagger/swagger.json
 
 	bin/swagger-order --output=api/swagger/swagger.json api/swagger/swagger.json
@@ -97,12 +111,8 @@ gen-api:            ## Generate PMM API
 		api/alerting/v1/json/v1.json \
 		api/advisors/v1/json/v1.json \
 		api/backup/v1/json/v1.json \
-		api/dump/v1/json/v1.json \
-		api/management/v1/azure/json/azure.json \
-		api/role/v1/json/v1.json \
-		api/management/v1/agent/json/agent.json \
-		api/management/v1/node/json/node.json \
-		api/management/v1/service/json/service.json \
+		api/dump/v1beta1/json/v1beta1.json \
+		api/accesscontrol/v1beta1/json/v1beta1.json \
 		api/qan/v1/json/v1.json \
 		api/platform/v1/json/v1.json
 
@@ -132,7 +142,21 @@ clean: clean_swagger  ## Remove generated files
 	find api -name '*.pb.gw.go' -print -delete
 	find api -name '*.validate.go' -print -delete
 
-	for API in api/agentlocal/v1 api/server/v1 api/inventory/v1 api/management/v1 api/actions/v1 api/alerting/v1 api/advisors/v1 api/backup/v1 api/dump/v1 api/role/v1 api/management/v1/agent api/management/v1/node api/management/v1/service api/qan/v1 api/platform/v1 ; do \
+	SPECS="\
+		api/agentlocal/v1 \
+		api/server/v1 \
+		api/user/v1 \
+		api/inventory/v1 \
+		api/management/v1 \
+		api/actions/v1 \
+		api/alerting/v1 \
+		api/advisors/v1 \
+		api/backup/v1 \
+		api/dump/v1beta1 \
+		api/accesscontrol/v1beta1 \
+		api/qan/v1 \
+		api/platform/v1"; \
+	for API in $$SPECS; do \
 		rm -fr $$API/json/client $$API/json/models $$API/json/$$(basename $$API).json ; \
 	done
 	rm -f api/swagger/swagger.json api/swagger/swagger-dev.json
@@ -145,8 +169,8 @@ api-test:             ## Run API tests on dev env.
 
 check:                ## Run required checkers and linters
 	bin/buf lint -v api
-	LOG_LEVEL=error bin/golangci-lint run ; \
-	bin/go-sumtype ./... ; \
+	LOG_LEVEL=error bin/golangci-lint run
+	bin/go-sumtype ./...
 	bin/go-consistent -pedantic ./...
 
 check-license:          ## Run license header checks against source files

admin/README.md

@@ -9,7 +9,7 @@ git, make, curl, go, gcc, pmm-server, pmm-agent
 
 ## Local setup
 ### To run pmm-admin commands
-- Run [pmm-server docker container](https://hub.docker.com/r/percona/pmm-server) or [pmm-managed](https://github.com/percona/pmm-managed).  
+- Run [pmm-server docker container](https://hub.docker.com/r/percona/pmm-server) or [pmm-managed](https://github.com/percona/pmm/tree/main/managed).  
 - Run pmm-agent `cd ../agent`.
 - Run pmm-admin commands.
     ```shell script
@@ -20,6 +20,7 @@ You should see something like this
  ```shell script
 Agent ID: fcbe3cb4-a95a-43f4-aef5-c3494caa5132
 Node ID : 77be6b4d-a1d9-4687-8fae-7acbaee7db47
+Node name: mysql-server-test-1
 
 PMM Server:
         URL    : https://127.0.0.1:443/
@@ -31,10 +32,10 @@ PMM-agent:
         Latency   : 211.026µs
 
 Agents:
-        3329a405-8a5d-4414-9890-b6ae4209e0cc NODE_EXPORTER RUNNING
+        3329a405-8a5d-4414-9890-b6ae4209e0cc NODE_EXPORTER RUNNING 40001
 ```
 It means that everything works.
 
 ## Testing
-pmm-admin doesn't require setting-up environment.  
+pmm-admin doesn't require setting-up an environment.  
 Run `make test` to run tests. 

admin/commands/annotation.go

@@ -17,6 +17,7 @@ package commands
 import (
 	"strings"
 
+	"github.com/AlekSi/pointer"
 	"github.com/pkg/errors"
 
 	"github.com/percona/pmm/admin/agentlocal"
@@ -77,9 +78,7 @@ func (cmd *AnnotationCommand) getCurrentNode() (*nodes.GetNodeOKBody, error) {
 	}
 
 	params := &nodes.GetNodeParams{
-		Body: nodes.GetNodeBody{
-			NodeID: status.NodeID,
-		},
+		NodeID:  status.NodeID,
 		Context: Ctx,
 	}
 
@@ -109,9 +108,7 @@ func (cmd *AnnotationCommand) getCurrentNodeAllServices() ([]string, error) {
 	}
 
 	params := &services.ListServicesParams{
-		Body: services.ListServicesBody{
-			NodeID: status.NodeID,
-		},
+		NodeID:  pointer.ToString(status.NodeID),
 		Context: Ctx,
 	}
 

admin/commands/base_test.go

@@ -86,7 +86,7 @@ func TestCredentials(t *testing.T) {
 		// Test reading is OK
 		t.Parallel()
 		creds, _ := ReadFromSource(credSource)
-		assert.Equal(t, creds.Username, "testuser")
+		assert.Equal(t, "testuser", creds.Username)
 	})
 
 	t.Run("Executing", func(t *testing.T) {

admin/commands/inventory/list_agents.go

@@ -108,16 +108,12 @@ func (cmd *ListAgentsCommand) RunCmd() (commands.Result, error) {
 		return nil, err
 	}
 
-	filters := agents.ListAgentsBody{
-		PMMAgentID: cmd.PMMAgentID,
-		ServiceID:  cmd.ServiceID,
-		NodeID:     cmd.NodeID,
-		AgentType:  agentType,
-	}
-
 	params := &agents.ListAgentsParams{
-		Body:    filters,
-		Context: commands.Ctx,
+		PMMAgentID: pointer.ToString(cmd.PMMAgentID),
+		ServiceID:  pointer.ToString(cmd.ServiceID),
+		NodeID:     pointer.ToString(cmd.NodeID),
+		AgentType:  agentType,
+		Context:    commands.Ctx,
 	}
 	agentsRes, err := client.Default.AgentsService.ListAgents(params)
 	if err != nil {

admin/commands/inventory/list_nodes.go

@@ -71,8 +71,8 @@ func (cmd *ListNodesCommand) RunCmd() (commands.Result, error) {
 	}
 
 	params := &nodes.ListNodesParams{
-		Body:    nodes.ListNodesBody{NodeType: nodeType},
-		Context: commands.Ctx,
+		NodeType: nodeType,
+		Context:  commands.Ctx,
 	}
 	result, err := client.Default.NodesService.ListNodes(params)
 	if err != nil {

admin/commands/inventory/list_services.go

@@ -19,6 +19,8 @@ import (
 	"net"
 	"strconv"
 
+	"github.com/AlekSi/pointer"
+
 	"github.com/percona/pmm/admin/commands"
 	"github.com/percona/pmm/api/inventory/v1/json/client"
 	services "github.com/percona/pmm/api/inventory/v1/json/client/services_service"
@@ -85,15 +87,11 @@ func (cmd *ListServicesCommand) RunCmd() (commands.Result, error) {
 		return nil, err
 	}
 
-	filters := services.ListServicesBody{
-		ExternalGroup: cmd.ExternalGroup,
-		NodeID:        cmd.NodeID,
-		ServiceType:   serviceType,
-	}
-
 	params := &services.ListServicesParams{
-		Body:    filters,
-		Context: commands.Ctx,
+		NodeID:        pointer.ToString(cmd.NodeID),
+		ExternalGroup: pointer.ToString(cmd.ExternalGroup),
+		ServiceType:   serviceType,
+		Context:       commands.Ctx,
 	}
 	result, err := client.Default.ServicesService.ListServices(params)
 	if err != nil {

admin/commands/inventory/remove_agent.go

@@ -15,6 +15,8 @@
 package inventory
 
 import (
+	"github.com/AlekSi/pointer"
+
 	"github.com/percona/pmm/admin/commands"
 	"github.com/percona/pmm/api/inventory/v1/json/client"
 	agents "github.com/percona/pmm/api/inventory/v1/json/client/agents_service"
@@ -41,10 +43,8 @@ type RemoveAgentCommand struct {
 // RunCmd executes the RemoveAgentCommand and returns the result.
 func (cmd *RemoveAgentCommand) RunCmd() (commands.Result, error) {
 	params := &agents.RemoveAgentParams{
-		Body: agents.RemoveAgentBody{
-			AgentID: cmd.AgentID,
-			Force:   cmd.Force,
-		},
+		AgentID: cmd.AgentID,
+		Force:   pointer.ToBool(cmd.Force),
 		Context: commands.Ctx,
 	}
 	_, err := client.Default.AgentsService.RemoveAgent(params)

admin/commands/inventory/remove_node.go

@@ -15,6 +15,8 @@
 package inventory
 
 import (
+	"github.com/AlekSi/pointer"
+
 	"github.com/percona/pmm/admin/commands"
 	"github.com/percona/pmm/api/inventory/v1/json/client"
 	nodes "github.com/percona/pmm/api/inventory/v1/json/client/nodes_service"
@@ -41,10 +43,8 @@ type RemoveNodeCommand struct {
 // RunCmd runs the command for RemoveNodeCommand.
 func (cmd *RemoveNodeCommand) RunCmd() (commands.Result, error) {
 	params := &nodes.RemoveNodeParams{
-		Body: nodes.RemoveNodeBody{
-			NodeID: cmd.NodeID,
-			Force:  cmd.Force,
-		},
+		NodeID:  cmd.NodeID,
+		Force:   pointer.ToBool(cmd.Force),
 		Context: commands.Ctx,
 	}
 	_, err := client.Default.NodesService.RemoveNode(params)