From 6d0b02bdcab1d18ef4a91591a31683c8270ebc00 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Mon, 1 Apr 2024 01:19:42 +0000 Subject: [PATCH] PMM-12899 extract server rpm script --- build/local/build.sh | 82 ++++++++++++++++++++------- build/scripts/build-rpmbuild-docker | 11 ---- build/scripts/build-server-docker | 18 +++--- build/scripts/build-server-rpm | 12 ++-- build/scripts/build-server-rpm-script | 40 +++++++++++++ 5 files changed, 115 insertions(+), 48 deletions(-) delete mode 100755 build/scripts/build-rpmbuild-docker create mode 100755 build/scripts/build-server-rpm-script diff --git a/build/local/build.sh b/build/local/build.sh index cba72f27df..80caaa0d2d 100755 --- a/build/local/build.sh +++ b/build/local/build.sh @@ -1,7 +1,38 @@ #!/bin/bash -e -# Update submodules and PR branches -/bin/bash $(dirname $0)/update.sh +# Define global variables +NO_UPDATE=0 +NO_CLIENT=0 +NO_CLIENT_DOCKER=0 +NO_SERVER_RPM=0 + +while test "$#" -gt 0; do + case "$1" in + --no-update) + NO_UPDATE=1 + ;; + --no-client) + NO_CLIENT=1 + ;; + --no-client-docker) + NO_CLIENT_DOCKER=1 + ;; + --no-server-rpm) + NO_SERVER_RPM=1 + ;; + *) + echo "Unknown argument: $1" + echo "Usage: $0 [--no-update] [--no-client] [--no-client-docker] [--no-server-rpm]" + exit 1 + ;; + esac + shift +done + +if [ "$NO_UPDATE" -eq 0 ]; then + # Update submodules and PR branches + /bin/bash $(dirname $0)/update.sh +fi get_branch_name() { local module="${1:-}" @@ -33,7 +64,7 @@ echo $pmm_ui_tests_commit > pmmUITestsCommitSha fb_commit_sha=$(git rev-parse HEAD) echo $fb_commit_sha > fbCommitSha -# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:ol9`. +# We use a special docker image to build various PMM artifacts - `perconalab/rpmbuild:3`. # Important: the docker container's user need to be able to write to these directories. # The docker container's user is `builder` with uid 1000 and gid 1000. You need to make sure # that the directories we create on the host are owned by a user with same uid and gid. @@ -51,22 +82,26 @@ export RPMBUILD_DOCKER_IMAGE=perconalab/rpmbuild:3 # RAM: 16GB # OS: Ubuntu 22.04.1 LTS -# Build client source: 4m39s from scratch, 0m27s using cache -"$PATH_TO_SCRIPTS/build-client-source" +if [ "$NO_CLIENT" -eq 0 ]; then + # Build client source: 4m39s from scratch, 0m27s using cache + "$PATH_TO_SCRIPTS/build-client-source" -# Build client binary: ??? from scratch, 0m20s using cache -"$PATH_TO_SCRIPTS/build-client-binary" + # Build client binary: ??? from scratch, 0m20s using cache + "$PATH_TO_SCRIPTS/build-client-binary" -# Building client source rpm takes 13s (caching does not apply) -"$PATH_TO_SCRIPTS/build-client-srpm" + # Building client source rpm takes 13s (caching does not apply) + "$PATH_TO_SCRIPTS/build-client-srpm" -# Building client rpm takes 1m40s -"$PATH_TO_SCRIPTS/build-client-rpm" + # Building client rpm takes 1m40s + "$PATH_TO_SCRIPTS/build-client-rpm" +fi # Building client docker image takes 17s GIT_COMMIT=$(git rev-parse HEAD | head -c 8) export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT} -"$PATH_TO_SCRIPTS/build-client-docker" +if [ "$NO_CLIENT_DOCKER" -eq 0 ] || [ "$NO_CLIENT" -eq 0 ]; then + "$PATH_TO_SCRIPTS/build-client-docker" +fi # Building PMM CLient locally (non-CI, i.e. non-Jenkins) # total time: 6m26s - build from scratch, no initial cache @@ -79,20 +114,23 @@ export DOCKER_CLIENT_TAG=local/pmm-client:${GIT_COMMIT} export RPM_EPOCH=1 export RPMBUILD_DIST="el9" -"$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards -"$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm -"$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm -"$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm -"$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump -"$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm -# 3rd-party -"$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics -"$PATH_TO_SCRIPTS/build-server-rpm" grafana +if [ "$NO_SERVER_RPM" -eq 0 ]; then + "$PATH_TO_SCRIPTS/build-server-rpm" percona-dashboards grafana-dashboards + "$PATH_TO_SCRIPTS/build-server-rpm" pmm-managed pmm + "$PATH_TO_SCRIPTS/build-server-rpm" percona-qan-api2 pmm + "$PATH_TO_SCRIPTS/build-server-rpm" pmm-update pmm + "$PATH_TO_SCRIPTS/build-server-rpm" pmm-dump + "$PATH_TO_SCRIPTS/build-server-rpm" vmproxy pmm + + # 3rd-party + "$PATH_TO_SCRIPTS/build-server-rpm" victoriametrics + "$PATH_TO_SCRIPTS/build-server-rpm" grafana +fi export DOCKER_TAG=local/pmm-server:${GIT_COMMIT} export RPMBUILD_DIST=el9 -export DOCKERFILE=Dockerfile.el9.local +export DOCKERFILE=Dockerfile.el9 ${PATH_TO_SCRIPTS}/build-server-docker # Clean up temporary files diff --git a/build/scripts/build-rpmbuild-docker b/build/scripts/build-rpmbuild-docker deleted file mode 100755 index 77706f3621..0000000000 --- a/build/scripts/build-rpmbuild-docker +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o xtrace - -. $(dirname $0)/vars - -root_dir=$(cd $(dirname $0)/../..; pwd -P) -docker build --squash -f ${docker_file} -t rpmbuild:local ${root_dir}/build/docker/rpmbuild/ - -# vim: expandtab shiftwidth=4 tabstop=4 diff --git a/build/scripts/build-server-docker b/build/scripts/build-server-docker index 421a4dd688..8cb2f70bfc 100755 --- a/build/scripts/build-server-docker +++ b/build/scripts/build-server-docker @@ -32,15 +32,15 @@ if [ -z "${DOCKER_TAG}" ]; then DOCKER_TAG=perconalab/pmm-server-fb:${full_pmm_version} fi -IMAGE_VERSION=`echo $DOCKER_TAG | cut -d ':' -f2` - -docker build --build-arg BUILD_DATE="`date --rfc-3339=seconds`" \ - --build-arg VERSION="$IMAGE_VERSION" \ - --squash \ - --no-cache \ - -f ${docker_root}/${docker_file} \ - -t ${DOCKER_TAG} \ - ${docker_root}/ +IMAGE_VERSION=$(echo "$DOCKER_TAG" | cut -d ':' -f2) + +docker buildx build \ + --build-arg BUILD_DATE=$(date --rfc-3339=seconds) \ + --build-arg VERSION="$IMAGE_VERSION" \ + --progress plain \ + -f ${docker_root}/${docker_file} \ + -t ${DOCKER_TAG} \ + ${docker_root} if [ -n "${PUSH_DOCKER}" ]; then mkdir -p $(dirname ${docker_tag_file}) diff --git a/build/scripts/build-server-rpm b/build/scripts/build-server-rpm index 06ae02326e..031aa7707e 100755 --- a/build/scripts/build-server-rpm +++ b/build/scripts/build-server-rpm @@ -24,7 +24,7 @@ is_build_needed() { local spec_name=$1 local rpm_version=$2 local packages= - local cache_dir=RELEASE + local s3_cache_dir=RELEASE # Structure of S3 build cache # s3://pmm-build-cache/PR-BUILDS/7 - el7 @@ -36,13 +36,13 @@ is_build_needed() { # RPM_EPOCH is set to 1 for Feature Builds if [ "$RPM_EPOCH" = 1 ]; then - cache_dir=PR-BUILDS + s3_cache_dir=PR-BUILDS fi if command -v aws &> /dev/null; then aws s3 sync \ --region us-east-2 \ --no-sign-request \ - s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \ + s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \ ${rpms_dir}/${spec_name}-${rpm_version} fi @@ -102,7 +102,7 @@ build() { local rpm_version=$(get_rpm_version "${spec_name}") local CH_VERSION=${rpm_version%-*} local CH_TAG=${rpm_version#*-} - local cache_dir=RELEASE + local s3_cache_dir=RELEASE local volume_mounts="-v ${rpmbuild_dir}/SOURCES:/home/builder/rpm/SOURCES -v ${rpms_dir}:/home/builder/rpm/RPMS" if ! is_build_needed "${spec_name}" "${rpm_version}"; then @@ -170,12 +170,12 @@ build() { fi if [ "$RPM_EPOCH" = 1 ]; then - cache_dir=PR-BUILDS + s3_cache_dir=PR-BUILDS fi aws s3 sync \ --region us-east-2 \ ${rpms_dir}/${spec_name}-${rpm_version} \ - s3://pmm-build-cache/${cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \ + s3://pmm-build-cache/${s3_cache_dir}/${rpmbuild_dist}/${spec_name}-${rpm_version} \ || : echo "Finished building Server RPMs, spec_name: ${spec_name}.spec, repo_name: ${repo_name}" diff --git a/build/scripts/build-server-rpm-script b/build/scripts/build-server-rpm-script new file mode 100755 index 0000000000..cece634cdc --- /dev/null +++ b/build/scripts/build-server-rpm-script @@ -0,0 +1,40 @@ +#!/bin/bash + +set -o errexit +set -o xtrace + +. $(dirname $0)/vars + +sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES /home/builder/.cache + +# Add 'Epoch' to spec file to prevent update of rpms which are built in PR build +if [ ${RPM_EPOCH} = 1 ]; then + sed -i '/^Version:.*/i Epoch: 1' /home/builder/rpm/SOURCES/${spec_name}.spec +fi + +rm -rf /home/builder/rpm/RPMS/${spec_name}-* + +printf '[local]\nname=local\nbaseurl=file:///home/builder/rpm/RPMS\ngpgcheck=0\nenabled=1\n' \ + | sudo tee /etc/yum.repos.d/local.repo + +until /usr/bin/createrepo_c --update /home/builder/rpm/RPMS; do + echo waiting + sleep 1 +done + +# Only these two specs have build dependencies +if [[ ${spec_name} =~ ^grafana$|^percona-dashboards$ ]]; then + sleep 5s + sudo yum-builddep -y SOURCES/${spec_name}.spec +fi + +spectool -C SOURCES -g SOURCES/${spec_name}.spec +rpmbuild --define '_rpmdir %{_topdir}/RPMS/${spec_name}-${rpm_version}' \ + --define 'dist .${rpmbuild_dist}' \ + --define 'debug_package %{nil}' \ + -ba SOURCES/${spec_name}.spec + +rm -f SOURCES/${spec_name}.spec* +sudo chown -R builder:builder /home/builder/rpm/RPMS /home/builder/rpm/SOURCES + +# vim: expandtab shiftwidth=4 tabstop=4