PMM-13487 Ability to build PMM locally

.devcontainer/setup.py

@@ -41,7 +41,7 @@ def install_go():
     """Installs Go toolchain."""
 
     run_commands([
-        "curl -sS https://raw.githubusercontent.com/travis-ci/gimme/v1.5.5/gimme -o /usr/local/bin/gimme",
+        "curl -sS https://raw.githubusercontent.com/travis-ci/gimme/v1.5.6/gimme -o /usr/local/bin/gimme",
         "chmod +x /usr/local/bin/gimme"
     ])
 

.github/workflows/main.yml

@@ -161,8 +161,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#00FF00"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"
@@ -179,8 +179,8 @@ jobs:
     env:
       SLACK_WEBHOOK: ${{ secrets.SLACK_TOKEN_PMM_CI }}
       SLACK_CHANNEL: "pmm-ci"
-      SLACK_USERNAME: "PR pipelines bot"
-      SLACK_ICON_EMOJI: ":chestnut:"
+      SLACK_USERNAME: "Build bot"
+      SLACK_ICON_EMOJI: ":octocat:"
       SLACK_COLOR: "#FF0000"
       SLACK_TITLE: "Finished ${{ github.event.repository.name }} workflow"
       SLACK_MESSAGE: "Workflow failed: ${{ github.event.inputs.repo || github.repository }}:${{ github.event.inputs.branch || github.head_ref }}"

.gitignore

@@ -1,46 +1,42 @@
-/.idea/
-/.vscode/
-*.iml
-
-/api/nginx/*.pem
-bin/
+.idea/
+.vscode/
+*.sw[o,p]
 
 # System Files
 .DS_Store
 
-cover.out
-crosscover.out
+# development
+.modules/
+bin/
+dist/
+.env
+.netrc
+pmm-agent-dev.yaml
+compose.yml
+ci.yml
+sbom.json
+*.log
 
-agent/testdata/
+api/nginx/*.pem
+api-tests/pmm-api-tests-output.txt
+api-tests/pmm-api-tests-junit-report.xml
 
+agent/testdata/
 agent/agents/mysql/slowlog/parser/corpus/
 agent/agents/mysql/slowlog/parser/crashers/
 agent/agents/mysql/slowlog/parser/suppressions/
 agent/agents/mysql/slowlog/parser/parser-fuzz.zip
-
 update/*.runcover.out
 update/*.test
 
-fuzzing/
-*-fuzz.zip
-*.bench
-
-backoff.png
-pmm-agent-dev.yaml
-compose.yml
-
-/qan-api2/logs/
-
-# ViM temporary files
-*.sw[o,p]
-
-.env
-.netrc
-
 cli-tests/node_modules/
 cli-tests/playwright-report/
 
-api-tests/pmm-api-tests-output.txt
-api-tests/pmm-api-tests-junit-report.xml
+qan-api2/logs/
 
-packer.log
+cover.out
+crosscover.out
+
+fuzzing/
+*-fuzz.zip
+*.bench

Makefile.devcontainer

@@ -43,15 +43,12 @@ run-vmproxy: release-vmproxy
 
 run-qan: release-qan
 	supervisorctl stop qan-api2
-	cp $(PMM_RELEASE_PATH)/qan-api2 /usr/sbin/percona-qan-api2
+	cp $(PMM_RELEASE_PATH)/qan-api2 /usr/sbin/pmm-qan-api
 	echo -n > /srv/logs/qan-api2.log
 	supervisorctl start qan-api2
 
 run-all: run-agent run-managed       ## Run pmm-managed and pmm-agent
 
-run:                                 ## Deprecated
-	echo "Deprecated: please use run-all"
-
 # TODO https://jira.percona.com/browse/PMM-3484, see maincover_test.go
 # run-race-cover: install-race    ## Run pmm-managed with race detector and collect coverage information.
 # 	go test -coverpkg="github.com/percona/pmm/managed/..." \

Makefile.include

@@ -30,16 +30,16 @@ gen: clean            ## Generate files
 	make -C admin gen
 	make -C managed gen
 
-	make gen-mocks      ## Generate mocks
+	make gen-mocks
 
 	make format
 	make format ## TODO: One formatting run is not enough, figure out why.
 	go install -v ./...
 
-clean:               ## Remove generated files
+clean:                ## Remove generated files
 	make -C api clean
 
-gen-mocks:
+gen-mocks:            ## Generate mocks
 	find . -name mock_*.go -delete
 	./bin/mockery --config .mockery.yaml
 
@@ -55,7 +55,7 @@ check:                ## Run required checkers and linters
 	bin/go-sumtype ./...
 	bin/go-consistent -pedantic ./...
 
-check-license:          ## Run license header checks against source files
+check-license:        ## Run license header checks against source files
 	bin/license-eye -c .licenserc.yaml header check
 
 check-all: check-license check    ## Run golangci linter to check for changes against main
@@ -69,5 +69,10 @@ format:               ## Format source code
 	bin/goimports -local github.com/percona/pmm -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/pmm)" $(FILES)
 
+tidy:                 ## Tidy up the go mod files
+	@echo "==> Tidy up submodules"
+	@go mod tidy
+	@cd tools && go mod tidy
+
 serve:                ## Serve API documentation with nginx
 	nginx -p . -c api/nginx/nginx.conf

admin/Makefile

@@ -11,6 +11,10 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-)
 PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
+
+ifdef NO_VCS
+	NO_VCS = -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -31,8 +35,8 @@ clean:                          ## Remove generated files
 	find . -name *_reform.go -delete
 
 release:                        ## Build pmm-admin release binary
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
-	env CGO_ENABLED=0 go build -v $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm-admin ./cmd/pmm-admin/
+	env CGO_ENABLED=0 go build -v $(NO_VCS) $(LD_FLAGS) -o $(PMM_RELEASE_PATH)/pmm ./cmd/pmm/
 
 install:                        ## Install pmm & pmm-admin binary
 	go build -v $(LD_FLAGS) -o $(GOBIN)/pmm-admin ./cmd/pmm-admin/

agent/Makefile

@@ -11,7 +11,11 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-)
 PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
-PMM_DEV_SERVER_PORT ?= 443
+PMM_DEV_SERVER_PORT ?= 8443
+
+ifdef NO_VCS
+	NO_VCS = -buildvcs=false
+endif
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif
@@ -25,8 +29,8 @@ VERSION_FLAGS = -X 'github.com/percona/pmm/version.ProjectName=pmm-agent' \
 				-X 'github.com/percona/pmm/version.Branch=$(PMM_RELEASE_BRANCH)'
 
 release:                        ## Build static pmm-agent release binary (Linux only)
-	env CGO_ENABLED=1 go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
-	go build -v -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
+	env CGO_ENABLED=1 go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent
+	go build -v $(NO_VCS) -ldflags "-extldflags '-static' $(VERSION_FLAGS)" -tags 'osusergo netgo static_build' -o $(PMM_RELEASE_PATH)/pmm-agent-entrypoint ./cmd/pmm-agent-entrypoint
 	$(PMM_RELEASE_PATH)/pmm-agent --version
 	ldd $(PMM_RELEASE_PATH)/pmm-agent 2>&1 | grep -Fq 'not a dynamic executable'
 

build.sh

@@ -0,0 +1 @@
+./build/local/build.sh

build/Makefile

@@ -8,8 +8,7 @@ export CENTOS_ISO := 2004.01
 fetch:
 	mkdir -p ${PACKER_CACHE_DIR}/box
 	test -f ${PACKER_CACHE_DIR}/id_rsa_vagrant \
-	    || curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant \
-		-o ${PACKER_CACHE_DIR}/id_rsa_vagrant
+		|| curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant -o ${PACKER_CACHE_DIR}/id_rsa_vagrant
 	chmod 600 ${PACKER_CACHE_DIR}/id_rsa_vagrant
 
 	# Add the box using Vagrant

build/ansible/roles/pmm-images/tasks/main.yml

@@ -88,13 +88,14 @@
   yum:
     name:
       - percona-grafana
-      - percona-victoriametrics
-      - percona-qan-api2
       - percona-dashboards
+      - pmm-victoriametrics
+      - pmm-qan-api
       - pmm-managed
+      - pmm-ui
       - pmm-update
       - pmm-dump
-      - vmproxy
+      - pmm-vmproxy
     state: installed
     enablerepo: local
 

build/docker/client/Dockerfile.el9

@@ -0,0 +1,62 @@
+FROM redhat/ubi9-minimal AS base
+
+RUN microdnf install shadow-utils jq tar -y
+
+RUN groupadd -g 1002 pmm-agent && \
+    useradd -u 1002 -r -g pmm-agent -s /sbin/nologin \
+            -d /usr/local/percona/pmm \
+            -c "PMM 3.x Client User" pmm-agent
+
+FROM redhat/ubi9-micro
+
+ARG VERSION
+ARG BUILD_DATE
+
+COPY LICENSE /licenses/
+COPY --from=base --chown=0:0 /etc/passwd /etc/passwd
+COPY --from=base --chown=0:0 /etc/group /etc/group
+
+# Install jq along with its dependencies
+COPY --from=base --chown=0:0 /usr/bin/jq /usr/bin/
+COPY --from=base --chown=0:0 /usr/bin/tar /usr/bin/
+COPY --from=base --chown=0:0 /usr/bin/curl /usr/bin/
+
+COPY --from=base --chown=0:0 /usr/lib64/libjq.so.1.0.4 /lib64/libjq.so.1
+COPY --from=base --chown=0:0 /usr/lib64/libonig.so.5.1.0 /lib64/libonig.so.5
+
+# Dependencies of curl
+COPY --from=base --chown=0:0 /usr/lib64/libkeyutils.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libz.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libkrb5support.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libkrb5.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libk5crypto.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libgssapi_krb5.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libcom_err.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libssl.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libcrypto.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libcurl.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libnghttp2.so* /lib64/
+COPY --from=base --chown=0:0 /usr/lib64/libresolv.so* /lib64/
+
+RUN install -d -o pmm-agent -g pmm-agent -m 0775 /usr/local/percona/pmm
+
+ADD pmm-client.tar.gz /tmp/
+
+RUN cd /tmp/pmm-client-* \
+    && PMM_USER=pmm-agent PMM_GROUP=root ./install_tarball \
+    && cd /tmp \
+    && rm -rf /tmp/pmm-client*
+
+LABEL org.opencontainers.image.created=${BUILD_DATE}
+LABEL org.opencontainers.image.licenses=Apache-2.0
+LABEL org.opencontainers.image.title="Percona Monitoring and Management Client"
+LABEL org.opencontainers.image.vendor=Percona
+LABEL org.opencontainers.image.version=${VERSION}
+LABEL org.opencontainers.image.source=https://github.com/percona/pmm/blob/main/build/docker/client/Dockerfile
+LABEL org.opencontainers.image.docs=https://docs.percona.com/percona-monitoring-and-management/setting-up/client/index.html#docker
+
+USER pmm-agent
+WORKDIR /usr/local/percona/pmm/
+ENV PATH=/usr/local/percona/pmm/bin/:$PATH
+
+ENTRYPOINT ["/usr/local/percona/pmm/bin/pmm-agent-entrypoint"]