PMM-12641 clean up build scripts (#2728)

* PMM-12641 clean up build scripts * PMM-12641 update build Makefile * PMM-12641 update build Makefile again * PMM-12641 provide comments * PMM-12641 update goreleaser version * PMM-12641 bump up goreleaser version again * PMM-12641 format json * PMM-12641 remove pmm.el9.json * PMM-12641 install percona-release * PMM-12641 print all installed gpg keys * PMM-12641 try to install ppg with yum * PMM-12641 final cleanup * PMM-12641 disable percona-release * PMM-12641 bump up software versions
percona · Feb 7, 2024 · 9310b83 · 9310b83
1 parent 86b90a9
commit 9310b83
Show file tree

Hide file tree

Showing 14 changed files with 50 additions and 434 deletions.
diff --git a/.gitignore b/.gitignore
@@ -35,6 +35,7 @@ compose.yml
 *.sw[o,p]
 
 .env
+.netrc
 
 cli-tests/node_modules/
 cli-tests/playwright-report/

diff --git a/build/Makefile b/build/Makefile
@@ -1,58 +1,33 @@
 export PACKER_CACHE_DIR := .cache
-export PACKER_VERSION := 1.8.2
-export CENTOS_ISO := 2004.01
+export PACKER_VERSION := 1.9.4
 
 ## ----------------- PACKER ------------------
 fetch:
-	mkdir -p ${PACKER_CACHE_DIR}/${CENTOS_ISO} || :
-	test -f ${PACKER_CACHE_DIR}/id_rsa_vagrant \
-	    || curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant \
-		-o ${PACKER_CACHE_DIR}/id_rsa_vagrant
-	chmod 600 ${PACKER_CACHE_DIR}/id_rsa_vagrant
-	test -f ${PACKER_CACHE_DIR}/${CENTOS_ISO}/CentOS7.ova \
-		|| wget --progress=dot:giga https://app.vagrantup.com/centos/boxes/7/versions/${CENTOS_ISO}/providers/virtualbox.box \
-		-O ${PACKER_CACHE_DIR}/${CENTOS_ISO}/CentOS7.ova
-	test -f ${PACKER_CACHE_DIR}/${CENTOS_ISO}/box.ovf \
-		|| tar -C ${PACKER_CACHE_DIR}/${CENTOS_ISO} -xvf ${PACKER_CACHE_DIR}/${CENTOS_ISO}/CentOS7.ova
-
-fetch-el9:
 	mkdir -p ${PACKER_CACHE_DIR}/box || :
 	test -f ${PACKER_CACHE_DIR}/id_rsa_vagrant \
 	    || curl -L https://raw.githubusercontent.com/hashicorp/vagrant/master/keys/vagrant \
 		-o ${PACKER_CACHE_DIR}/id_rsa_vagrant
 	chmod 600 ${PACKER_CACHE_DIR}/id_rsa_vagrant
 	test -f ${PACKER_CACHE_DIR}/box/oracle9.ova \
-		|| wget --progress=dot:giga  https://vagrantcloud.com/bento/boxes/oracle-9.0/versions/202207.20.0/providers/virtualbox.box -O ${PACKER_CACHE_DIR}/box/oracle9.ova
+		|| curl -fL https://vagrantcloud.com/bento/boxes/oracle-9.0/versions/202207.20.0/providers/virtualbox.box -o ${PACKER_CACHE_DIR}/box/oracle9.ova
 
 	# NOTE: image from vagrant registry is twice as large
 	test -f ${PACKER_CACHE_DIR}/box/box.ovf \
 		|| tar -C ${PACKER_CACHE_DIR}/box -xvf ${PACKER_CACHE_DIR}/box/oracle9.ova
 
 deps:
 	mkdir -p ${PACKER_CACHE_DIR} ~/bin || :
-	curl https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip -o ${PACKER_CACHE_DIR}/packer.zip
+	curl -fL https://releases.hashicorp.com/packer/${PACKER_VERSION}/packer_${PACKER_VERSION}_linux_amd64.zip -o ${PACKER_CACHE_DIR}/packer.zip
 	unzip -o ${PACKER_CACHE_DIR}/packer.zip -d ~/bin
 
 pmm-ovf: fetch
-	packer build -only virtualbox-ovf packer/pmm.json
-
-pmm-ovf-el9-rc: fetch-el9
-	sed -i 's|become_method: su|become_method: sudo|g' update/tasks/roles/postgres/tasks/main.yml && \
 	/usr/bin/packer build \
-			-var 'pmm_client_repos=original testing' \
-			-var 'pmm_client_repo_name=percona-testing-x86_64' \
-			-var 'pmm_server_repo=testing' \
-			-only virtualbox-ovf -color=false packer/pmm.el9.json \
-					| tee build.log
+			-only virtualbox-ovf -color=false packer/pmm.json | tee build.log
 
-pmm-ovf-el9-3-dev-latest: fetch-el9
-	sed -i 's|become_method: su|become_method: sudo|g' update/tasks/roles/postgres/tasks/main.yml && \
+# NOTE: no difference between rc and dev-latest (i.e. pmm-ovf) ATM
+pmm-ovf-rc: fetch
 	/usr/bin/packer build \
-			-var 'pmm_client_repos=original experimental' \
-			-var 'pmm_client_repo_name=percona-experimental-x86_64' \
-			-var 'pmm_server_repo=experimental' \
-			-only virtualbox-ovf -color=false packer/pmm.el9.json \
-					| tee build.log
+			-only virtualbox-ovf -color=false packer/pmm.json | tee build.log
 
 pmm-digitalocean:
 	packer build -only digitalocean -var 'single_disk=true' packer/pmm.json
@@ -62,11 +37,10 @@ pmm-azure:
 
 pmm-ami:
 	docker run --rm -v ${HOME}/.aws:/root/.aws -v `pwd`:/build -w /build \hashicorp/packer:${PACKER_VERSION} \
-	build -only amazon-ebs -color=false packer/pmm.json | tee build.log
-
+		build -only amazon-ebs -color=false packer/pmm.json | tee build.log
 ## ----------------- PACKER ------------------
 
-check:                          ## Run required checkers and linters
+check:                          ## Run required checks and linters
 	ansible-playbook --syntax-check ansible/pmm-docker/update.yml
 	ansible-playbook --check ansible/pmm-docker/update.yml
 	ansible-lint ansible/pmm-docker/update.yml
diff --git a/build/ansible/pmm-docker/update.yml b/build/ansible/pmm-docker/update.yml
@@ -111,7 +111,6 @@
       become: true
       become_user: pmm
       become_method: su
-      # changed_when: true
       with_nested:
         - - nginx
           - grafana
@@ -125,29 +124,6 @@
     - name: Check grafana logs
       shell: cat /srv/logs/grafana.log
 
-    - name: Fix grafana fields type
-      postgresql_query:
-        db: grafana
-        query: "{{ item }}"
-      loop:
-        - ALTER TABLE tag ALTER COLUMN key TYPE text;
-        - ALTER TABLE tag ALTER COLUMN value TYPE text;
-        - ALTER TABLE api_key ALTER COLUMN key TYPE text;
-        - ALTER TABLE api_key ALTER COLUMN name TYPE text;
-      when: not ansible_check_mode
-
-    - name: Change default admin id
-      postgresql_query:
-        db: grafana
-        query: UPDATE "user" SET id='1' WHERE login='admin';
-      when: not ansible_check_mode
-
-    # - name: Remove redundant packages
-    #   yum:
-    #     state: absent
-    #     name:
-    #       - logrotate # https://jira.percona.com/browse/PMM-7627
-
     # Regenerating pmm.ini and enabling pmm-update-perform-init
     - name: Generate new supervisor config
       command: pmm-managed-init

diff --git a/build/ansible/pmm/post-build-actions.yml b/build/ansible/pmm/post-build-actions.yml
@@ -5,8 +5,6 @@
   become: yes
   gather_facts: yes
   vars:
-    pmm_client_repos: "original testing"
-    pmm_client_repos_final: "original release"
     pmm_server_distribution: "docker"
 
   tasks:
@@ -20,12 +18,6 @@
         owner: pmm
         group: pmm
 
-    - name: Disable testing repo for pmm-client
-      command: percona-release disable {{ pmm_client_repos }}
-
-    - name: Enable release repo for pmm-client
-      command: percona-release enable {{ pmm_client_repos_final }}
-
     - name: Set up pmm-agent
       command: >
         pmm-agent setup

diff --git a/build/ansible/roles/pmm-images/tasks/main.yml b/build/ansible/roles/pmm-images/tasks/main.yml
@@ -1,38 +1,24 @@
 ---
-- name: Install the GPG key for repo.percona.com
-  rpm_key:
-    state: present
-    key: https://downloads.percona.com/downloads/RPM-GPG-KEY-percona
+# We use percona-release package to install the gpg keys and ppg-14
+# To workaround the package's incompatibility with RHEL9, we have to disable the gpg verification :(
+- name: Add percona-release package
+  yum:
+    name: https://repo.percona.com/yum/percona-release-latest.noarch.rpm
+    state: installed
+    disable_gpg_check: yes
 
-- name: Add PMM3 Server YUM repository
-  yum_repository:
-    name: pmm-server
-    description: PMM Server YUM repository - x86_64
-    baseurl: https://repo.percona.com/pmm3-components/yum/{{ pmm_server_repo }}/9/RPMS/x86_64/
-    gpgcheck: yes
-    enabled: yes
-    gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY
+- name: List installed gpg keys
+  command: ls -la /etc/pki/rpm-gpg
 
 # Local yum repo for building pmm server docker image in autobuild jobs
 - name: Add a local YUM repository
-  when: ansible_virtualization_type == "docker"
   yum_repository:
     name: local
     description: Local YUM repository - x86_64
     baseurl: file:///tmp/RPMS
     gpgcheck: no
     enabled: no
 
-# we use it for pmm-client (TODO we'll need to switch to pmm-client repo)
-# To workaround the package's incompatibility with RHEL9, we have to ignore errors :(
-# Error: `Failed to validate GPG signature for percona-release-1.0-27.noarch`
-# Despite the error, this will still install the repo and the GPG key
-- name: Add percona-release repository
-  yum:
-    name: https://repo.percona.com/yum/percona-release-latest.noarch.rpm
-    state: installed
-  ignore_errors: True
-
 - name: Update OS packages
   yum:
     name: "*"
@@ -47,41 +33,28 @@
       - python3.11-psycopg2
       - rsync
 
-- name: Install ansible-core and ansible collections
-  yum:
-    name:
-      - ansible-core
-      - ansible-collection-community-general
-      - ansible-collection-community-postgresql
-      - ansible-collection-ansible-posix
-    state: present
-
-- name: Create users and groups in docker container
-  block:
-    - name: Ensure groups exist with correct gid
-      group:
-        name: "{{ item.name }}"
-        gid: "{{ item.gid }}"
-      loop:
-        - { name: pmm, gid: 1000 }
-        - { name: nginx, gid: 1001 }
-        - { name: clickhouse, gid: 1002 }
-
-    # Note: nginx and clickhouse users will get removed in post-build.
-    - name: Create users
-      user:
-        name: "{{ item.name }}"
-        uid: "{{ item.uid }}"
-        home: "{{ item.home }}"
-        comment: "{{ item.comment }}"
-        shell: "{{ item.shell }}"
-        group: "{{ item.group }}"
-      loop:
-        - { name: pmm, uid: 1000, comment: "PMM Server", shell: "/usr/bin/bash", home: "/home/pmm", group: pmm, }
-        - { name: nginx, uid: 1001, comment: "Nginx user", shell: "/sbin/nologin", home: "/dev/null", group: nginx, }
-        - { name: clickhouse, uid: 1002, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/dev/null", group: clickhouse, }
-
-  when: ansible_virtualization_type == "docker"
+- name: Create groups
+  group:
+    name: "{{ item.name }}"
+    gid: "{{ item.gid }}"
+  loop:
+    - { name: pmm, gid: 1000 }
+    - { name: nginx, gid: 1001 }
+    - { name: clickhouse, gid: 1002 }
+
+# Note: nginx and clickhouse users will get removed in post-build.
+- name: Create users
+  user:
+    name: "{{ item.name }}"
+    uid: "{{ item.uid }}"
+    home: "{{ item.home }}"
+    comment: "{{ item.comment }}"
+    shell: "{{ item.shell }}"
+    group: "{{ item.group }}"
+  loop:
+    - { name: pmm, uid: 1000, comment: "PMM Server", shell: "/usr/bin/bash", home: "/home/pmm", group: pmm, }
+    - { name: nginx, uid: 1001, comment: "Nginx user", shell: "/sbin/nologin", home: "/dev/null", group: nginx, }
+    - { name: clickhouse, uid: 1002, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/dev/null", group: clickhouse, }
 
 - name: Create directories (mask 022)
   file: 
@@ -105,7 +78,6 @@
     mode: 0775
 
 - name: Create dirs
-  when: ansible_virtualization_type == "docker"
   file: 
     path: "{{ item }}"
     state: directory
@@ -125,8 +97,7 @@
       - pmm-dump
       - vmproxy
     state: installed
-    # line below is sed'ed by build-server-docker script
-    enablerepo: "pmm-server"
+    enablerepo: local
 
 - name: Create grafana config
   include_role:
@@ -148,9 +119,6 @@
   include_role:
     name: supervisord
 
-- name: Enable repo for pmm-client
-  command: percona-release enable {{ pmm_client_repos }}
-
 - name: Install pmm-client
   include_role:
     name: pmm-client

diff --git a/build/ansible/roles/pmm-images/vars/main.yml b/build/ansible/roles/pmm-images/vars/main.yml
diff --git a/build/ansible/roles/postgres/tasks/main.yml b/build/ansible/roles/postgres/tasks/main.yml
@@ -8,11 +8,10 @@
     enabled: yes
     gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY
 
-# we need the old postgres binary for the upgrade process
 - name: Install Postgres
   when:
     - not ansible_check_mode
-  dnf:
+  yum:
     name:
       - percona-postgresql14-server
       - percona-postgresql14-contrib
@@ -100,7 +99,7 @@
 
 - name: Upgrade grafana database to the latest schema
   command: grafana cli --homepath=/usr/share/grafana --config=/etc/grafana/grafana.ini admin data-migration encrypt-datasource-passwords
-  changed_when: True
+  changed_when: true
 
 - name: Stop Postgres 14 database without supervisor
   command: /usr/pgsql-14/bin/pg_ctl stop -D /srv/postgres14

diff --git a/build/docker/rpmbuild/Dockerfile b/build/docker/rpmbuild/Dockerfile
diff --git a/build/docker/rpmbuild/Dockerfile.el9 b/build/docker/rpmbuild/Dockerfile.el9
@@ -25,12 +25,12 @@ RUN yum update -y && \
     yum clean all && rm -rf /var/cache/yum
 
 # keep that format for easier search
-ENV GO_VERSION 1.21.3
-ENV GO_RELEASER_VERSION 1.21.2
+ENV GO_VERSION 1.21.7
+ENV GO_RELEASER_VERSION 1.24.0
 
 RUN if [ `uname -i` == "x86_64" ]; then ARCH=amd64; else ARCH=arm64; fi && \
-    wget --progress=dot:giga https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz -O /tmp/golang.tar.gz && \
-    wget --progress=dot:giga https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm -O /tmp/goreleaser.rpm && \
+    curl -fSsL -o /tmp/golang.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-${ARCH}.tar.gz && \
+    curl -fSsL -o /tmp/goreleaser.rpm https://github.com/goreleaser/goreleaser/releases/download/v${GO_RELEASER_VERSION}/goreleaser-${GO_RELEASER_VERSION}-1.`uname -i`.rpm && \
     tar -C /usr/local -xzf /tmp/golang.tar.gz && \
     yum install -y /tmp/goreleaser.rpm && \
     rm /tmp/golang.tar.gz /tmp/goreleaser.rpm
-Original file line number
+Diff line change
@@ Expand Up / @@ -35,6 +35,7 @@ compose.yml @@
     *.sw[o,p]
     .env
+    .netrc
     cli-tests/node_modules/
     cli-tests/playwright-report/
@@ Expand Down @@