diff --git a/.e2eignore b/.e2eignore new file mode 100644 index 000000000..b7442d7bb --- /dev/null +++ b/.e2eignore @@ -0,0 +1,12 @@ +.github/** +docs/** +code-of-conduct.md +CONTRIBUTING.md +README.md +.gitattributes +.gitignore +LICENSE +operator.png +kubernetes.svg +.e2eignore +release_versions \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index be0e7fbe8..32bf1d21d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -region='us-central1-a' +region="us-central1-a" testUrlPrefix="https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator" tests=[] @@ -13,8 +13,7 @@ void createCluster(String CLUSTER_SUFFIX) { gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE gcloud config set project $GCP_PROJECT gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true - - gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \ + gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias && \ kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$? if [ \${ret_val} -eq 0 ]; then break; fi ret_num=\$((ret_num + 1)) @@ -145,12 +144,12 @@ void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) { echo kubectl top pod --all-namespaces echo - kubectl get events --field-selector type!=Normal --all-namespaces + kubectl get events --field-selector type!=Normal --all-namespaces --sort-by=".lastTimestamp" echo "======================================================" """ } -TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |' +TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |' TestsReportXML = '\n' void makeReport() { @@ -158,9 +157,9 @@ void makeReport() { def startedTestAmount = 0 for (int i=0; i\n' + + sh """ + echo "${TestsReportXML}" > TestsReport.xml + """ } void clusterRunner(String cluster) { @@ -224,6 +227,7 @@ void runTest(Integer TEST_ID) { } catch (exc) { printKubernetesStatus("AFTER","$clusterSuffix") + echo "Test $testName has failed!" if (retryCount >= 1 || currentBuild.nextBuild != null) { currentBuild.result = 'FAILURE' return true @@ -241,9 +245,106 @@ void runTest(Integer TEST_ID) { } } -def skipBranchBuilds = true +void prepareNode() { + sh """ + sudo curl -s -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && sudo chmod +x /usr/local/bin/kubectl + kubectl version --client --output=yaml + + curl -fsSL https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz | sudo tar -C /usr/local/bin --strip-components 1 -xzf - linux-amd64/helm + + sudo curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -o /usr/local/bin/yq && sudo chmod +x /usr/local/bin/yq + sudo curl -fsSL https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux64 -o /usr/local/bin/jq && sudo chmod +x /usr/local/bin/jq + + sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOF +[google-cloud-cli] +name=Google Cloud CLI +baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOF + sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin + + curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense + + sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true + sudo percona-release enable-only tools + sudo yum install -y percona-xtrabackup-80 | true + """ +} + +boolean isManualBuild() { + def causes = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause') + return !causes.isEmpty() +} + +needToRunTests = true +void checkE2EIgnoreFiles() { + if (isManualBuild()) { + echo "This is a manual rebuild. Forcing pipeline execution." + return + } + + def e2eignoreFile = ".e2eignore" + if ( ! fileExists(e2eignoreFile) ) { + echo "No $e2eignoreFile file found. Proceeding with execution." + return + } + + def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()} + def lastProcessedCommitFile = "last-processed-commit.txt" + def lastProcessedCommitHash = "" + + def build = currentBuild.previousBuild + while (build != null) { + try { + echo "Checking previous build: #$build.number" + copyArtifacts(projectName: env.JOB_NAME, selector: specific("$build.number"), filter: lastProcessedCommitFile) + lastProcessedCommitHash = readFile(lastProcessedCommitFile).trim() + echo "Last processed commit hash: $lastProcessedCommitHash" + break + } catch (Exception e) { + echo "No $lastProcessedCommitFile found in build $build.number. Checking earlier builds." + } + build = build.previousBuild + } + + if (lastProcessedCommitHash == "") { + echo "This is the first run. Using merge base as the starting point for the diff." + changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n').findAll{it} + } else { + echo "Processing changes since last processed commit: $lastProcessedCommitHash" + changedFiles = sh(script: "git diff --name-only $lastProcessedCommitHash HEAD", returnStdout: true).trim().split('\n').findAll{it} + } + + echo "Excluded files: $excludedFiles" + echo "Changed files: $changedFiles" + + def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")} + needToRunTests = !changedFiles.every{changed -> excludedFilesRegex.any{regex -> changed ==~ regex}} + + if (needToRunTests) { + echo "Some changed files are outside of the e2eignore list. Proceeding with execution." + } else { + if (currentBuild.previousBuild?.result in ['FAILURE', 'ABORTED', 'UNSTABLE']) { + echo "All changed files are e2eignore files, and previous build was unsuccessful. Propagating previous state." + currentBuild.result = currentBuild.previousBuild?.result + error "Skipping execution as non-significant changes detected and previous build was unsuccessful." + } else { + echo "All changed files are e2eignore files. Aborting pipeline execution." + } + } + + sh """ + echo \$(git rev-parse HEAD) > $lastProcessedCommitFile + """ + archiveArtifacts "$lastProcessedCommitFile" +} + +def isPRJob = false if (env.CHANGE_URL) { - skipBranchBuilds = false + isPRJob = true } pipeline { @@ -262,16 +363,28 @@ pipeline { } options { disableConcurrentBuilds(abortPrevious: true) + copyArtifactPermission("$JOB_NAME/PR-*") } stages { + stage('Check Ignore Files') { + when { + expression { + isPRJob + } + } + steps { + checkE2EIgnoreFiles() + } + } stage('Prepare') { when { expression { - !skipBranchBuilds + isPRJob && needToRunTests } } steps { initTests() + prepareNode() script { if (AUTHOR_NAME == 'null') { AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", , returnStdout: true).trim() @@ -284,33 +397,6 @@ pipeline { } } } - sh """ - sudo curl -s -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && sudo chmod +x /usr/local/bin/kubectl - kubectl version --client --output=yaml - - curl -fsSL https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz | sudo tar -C /usr/local/bin --strip-components 1 -xzf - linux-amd64/helm - - sudo curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -o /usr/local/bin/yq && sudo chmod +x /usr/local/bin/yq - sudo curl -fsSL https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux64 -o /usr/local/bin/jq && sudo chmod +x /usr/local/bin/jq - - sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOF -[google-cloud-cli] -name=Google Cloud CLI -baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=0 -gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -EOF - sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin - - curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense - - sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true - sudo percona-release enable-only tools - sudo yum install -y percona-xtrabackup-80 | true - """ - withCredentials([file(credentialsId: 'cloud-secret-file', variable: 'CLOUD_SECRET_FILE')]) { sh ''' cp $CLOUD_SECRET_FILE e2e-tests/conf/cloud-secret.yml @@ -322,14 +408,14 @@ EOF stage('Build docker image') { when { expression { - !skipBranchBuilds + isPRJob && needToRunTests } } steps { withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) { sh ''' DOCKER_TAG=perconalab/percona-xtradb-cluster-operator:$VERSION - docker_tag_file='./results/docker/TAG' + docker_tag_file='results/docker/TAG' mkdir -p $(dirname ${docker_tag_file}) echo ${DOCKER_TAG} > "${docker_tag_file}" sg docker -c " @@ -339,7 +425,7 @@ EOF ./e2e-tests/build docker logout " - sudo rm -rf ./build + sudo rm -rf build ''' } stash includes: 'results/docker/TAG', name: 'IMAGE' @@ -349,7 +435,7 @@ EOF stage('GoLicenseDetector test') { when { expression { - !skipBranchBuilds + isPRJob && needToRunTests } } steps { @@ -376,7 +462,7 @@ EOF stage('GoLicense test') { when { expression { - !skipBranchBuilds + isPRJob && needToRunTests } } steps { @@ -396,7 +482,7 @@ EOF withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'GITHUB_TOKEN')]) { sh """ - golicense -plain ./percona-xtradb-cluster-operator \ + golicense -plain percona-xtradb-cluster-operator \ | grep -v 'license not found' \ | sed -r 's/^[^ ]+[ ]+//' \ | sort \ @@ -410,7 +496,7 @@ EOF stage('Run tests for operator') { when { expression { - !skipBranchBuilds + isPRJob && needToRunTests } } options { @@ -478,34 +564,31 @@ EOF slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}" } } - - if (env.CHANGE_URL && currentBuild.nextBuild == null) { - for (comment in pullRequest.comments) { - println("Author: ${comment.user}, Comment: ${comment.body}") - if (comment.user.equals('JNKPercona')) { - println("delete comment") - comment.delete() + if (needToRunTests) { + if (isPRJob && currentBuild.nextBuild == null) { + for (comment in pullRequest.comments) { + println("Author: ${comment.user}, Comment: ${comment.body}") + if (comment.user.equals('JNKPercona')) { + println("delete comment") + comment.delete() + } } + makeReport() + step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0]) + archiveArtifacts '*.xml' + + unstash 'IMAGE' + def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim() + TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n" + pullRequest.comment(TestsReport) } - makeReport() + deleteOldClusters("$CLUSTER_NAME") sh """ - echo "${TestsReportXML}" > TestsReport.xml + sudo docker system prune --volumes -af """ - step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0]) - archiveArtifacts '*.xml' - - unstash 'IMAGE' - def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim() - TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n" - pullRequest.comment(TestsReport) } + deleteDir() } - deleteOldClusters("$CLUSTER_NAME") - sh """ - sudo docker system prune --volumes -af - sudo rm -rf * - """ - deleteDir() } } }