From 617c34481cee1a8c408b9fca4f6b1a202214c6f3 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Thu, 5 Dec 2024 22:40:28 +0200
Subject: [PATCH 01/44] Enhance Jenkinsfile to skip builds for specified
non-trigger files
---
Jenkinsfile | 12 ++++++++++--
non-trigger-files.txt | 5 +++++
2 files changed, 15 insertions(+), 2 deletions(-)
create mode 100644 non-trigger-files.txt
diff --git a/Jenkinsfile b/Jenkinsfile
index 0ad3383f2..cc2a8a5b6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -409,8 +409,16 @@ EOF
}
stage('Run tests for operator') {
when {
- expression {
- !skipBranchBuilds
+ allOf {
+ not {
+ anyOf {
+ def excludedFiles = readFile("non-trigger-files.txt").split('\n').collect{it.trim()}
+ excludedFiles.each{changeset(it)}
+ }
+ }
+ expression {
+ !skipBranchBuilds
+ }
}
}
options {
diff --git a/non-trigger-files.txt b/non-trigger-files.txt
new file mode 100644
index 000000000..1fce30a32
--- /dev/null
+++ b/non-trigger-files.txt
@@ -0,0 +1,5 @@
+docs/**
+LICENSE
+operator.png
+README.md
+kubernetes.svg
\ No newline at end of file
From 9528209a63e05ceae3c66d27dc5e36ad0035632b Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Thu, 5 Dec 2024 23:26:26 +0200
Subject: [PATCH 02/44] Add logic to determine non-trigger files in Jenkins
pipeline
---
Jenkinsfile | 26 +++++++++++++++++++++-----
1 file changed, 21 insertions(+), 5 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index cc2a8a5b6..facc17f1e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -246,6 +246,8 @@ if (env.CHANGE_URL) {
skipBranchBuilds = false
}
+def nonTriggerFiles = false
+
pipeline {
environment {
CLOUDSDK_CORE_DISABLE_PROMPTS = 1
@@ -407,14 +409,28 @@ EOF
}
}
}
+ stage('Determine non-trigger files') {
+ steps {
+ script {
+ def changesetFile = "non-trigger-files.txt"
+ if (fileExists(changesetFile)) {
+ def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
+ def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
+ echo "Excluded files: ${excludedFiles}"
+ echo "Changed files: ${changedFiles}"
+
+ nonTriggerFiles = changedFiles.any { changed ->
+ excludedFiles.any { excluded -> changed ==~ excluded }
+ }
+ }
+ }
+ }
+ }
stage('Run tests for operator') {
when {
allOf {
- not {
- anyOf {
- def excludedFiles = readFile("non-trigger-files.txt").split('\n').collect{it.trim()}
- excludedFiles.each{changeset(it)}
- }
+ expression {
+ !nonTriggerFiles
}
expression {
!skipBranchBuilds
From e072a1072ba62187d5b65cea4536b36940b24dc6 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Thu, 5 Dec 2024 23:34:45 +0200
Subject: [PATCH 03/44] Fix logic to determine non-trigger files in Jenkins
pipeline
---
Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index facc17f1e..ec8996f64 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -419,7 +419,7 @@ EOF
echo "Excluded files: ${excludedFiles}"
echo "Changed files: ${changedFiles}"
- nonTriggerFiles = changedFiles.any { changed ->
+ nonTriggerFiles = changedFiles.every { changed ->
excludedFiles.any { excluded -> changed ==~ excluded }
}
}
From 461414f43406004ce1a0a2a837e43ca18390ebb2 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 00:04:39 +0200
Subject: [PATCH 04/44] Refactor Jenkinsfile to comment out non-trigger files
determination logic
---
Jenkinsfile | 44 +++++++++++++++++++++++++-------------------
1 file changed, 25 insertions(+), 19 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index ec8996f64..1b6a976f4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -409,28 +409,34 @@ EOF
}
}
}
- stage('Determine non-trigger files') {
- steps {
- script {
- def changesetFile = "non-trigger-files.txt"
- if (fileExists(changesetFile)) {
- def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
- def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
- echo "Excluded files: ${excludedFiles}"
- echo "Changed files: ${changedFiles}"
-
- nonTriggerFiles = changedFiles.every { changed ->
- excludedFiles.any { excluded -> changed ==~ excluded }
- }
- }
- }
- }
- }
+ // stage('Determine non-trigger files') {
+ // steps {
+ // script {
+ // def changesetFile = "non-trigger-files.txt"
+ // if (fileExists(changesetFile)) {
+ // def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
+ // def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
+ // echo "Excluded files: ${excludedFiles}"
+ // echo "Changed files: ${changedFiles}"
+
+ // nonTriggerFiles = changedFiles.every { changed ->
+ // excludedFiles.any { excluded -> changed ==~ excluded }
+ // }
+ // }
+ // }
+ // }
+ // }
stage('Run tests for operator') {
when {
allOf {
- expression {
- !nonTriggerFiles
+ not {
+ anyOf {
+ changeset "docs/**"
+ changeset "LICENSE"
+ changeset "operator.png"
+ changeset "README.md"
+ changeset "kubernetes.svg"
+ }
}
expression {
!skipBranchBuilds
From 524319681aa552b550aa698b25e0fca7d3b597ab Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 00:26:38 +0200
Subject: [PATCH 05/44] Add non-trigger-files.txt to changeset in Jenkinsfile
---
Jenkinsfile | 1 +
1 file changed, 1 insertion(+)
diff --git a/Jenkinsfile b/Jenkinsfile
index 1b6a976f4..38495c58b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -436,6 +436,7 @@ EOF
changeset "operator.png"
changeset "README.md"
changeset "kubernetes.svg"
+ changeset "non-trigger-files.txt"
}
}
expression {
From c80e42f5fc318359f2b62ce18764fdc5259d6c52 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 07:36:35 +0200
Subject: [PATCH 06/44] Refactor Jenkinsfile to restore non-trigger files
determination logic and update non-trigger-files.txt
---
Jenkinsfile | 45 ++++++++++++++++++-------------------------
non-trigger-files.txt | 3 ++-
2 files changed, 21 insertions(+), 27 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 38495c58b..9e7035a94 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -409,35 +409,28 @@ EOF
}
}
}
- // stage('Determine non-trigger files') {
- // steps {
- // script {
- // def changesetFile = "non-trigger-files.txt"
- // if (fileExists(changesetFile)) {
- // def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
- // def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
- // echo "Excluded files: ${excludedFiles}"
- // echo "Changed files: ${changedFiles}"
-
- // nonTriggerFiles = changedFiles.every { changed ->
- // excludedFiles.any { excluded -> changed ==~ excluded }
- // }
- // }
- // }
- // }
- // }
+ stage('Determine non-trigger files') {
+ steps {
+ script {
+ def changesetFile = "non-trigger-files.txt"
+ if (fileExists(changesetFile)) {
+ def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
+ def changedFiles = sh(script: "git diff --name-only origin/main", returnStdout: true).trim().split('\n')
+ echo "Excluded files: ${excludedFiles}"
+ echo "Changed files: ${changedFiles}"
+
+ nonTriggerFiles = changedFiles.every { changed ->
+ excludedFiles.any { excluded -> changed ==~ excluded }
+ }
+ }
+ }
+ }
+ }
stage('Run tests for operator') {
when {
allOf {
- not {
- anyOf {
- changeset "docs/**"
- changeset "LICENSE"
- changeset "operator.png"
- changeset "README.md"
- changeset "kubernetes.svg"
- changeset "non-trigger-files.txt"
- }
+ expression {
+ !nonTriggerFiles
}
expression {
!skipBranchBuilds
diff --git a/non-trigger-files.txt b/non-trigger-files.txt
index 1fce30a32..2600d5815 100644
--- a/non-trigger-files.txt
+++ b/non-trigger-files.txt
@@ -2,4 +2,5 @@ docs/**
LICENSE
operator.png
README.md
-kubernetes.svg
\ No newline at end of file
+kubernetes.svg
+non-trigger-files.txt
From 89e3e9d1faff8e60b6c68a2ffed6376d02b47fc2 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 08:08:55 +0200
Subject: [PATCH 07/44] Enhance non-trigger files logic in Jenkinsfile to use
regex for exclusion and improve logging
---
Jenkinsfile | 21 ++++++++++++++++++---
1 file changed, 18 insertions(+), 3 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 9e7035a94..979d1cea7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -415,15 +415,30 @@ EOF
def changesetFile = "non-trigger-files.txt"
if (fileExists(changesetFile)) {
def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
+
+ def convertGlobToRegex = { glob ->
+ glob.replace("**", ".*").replace("*", "[^/]*")
+ }
+ def excludedRegexes = excludedFiles.collect { convertGlobToRegex(it) }
+
def changedFiles = sh(script: "git diff --name-only origin/main", returnStdout: true).trim().split('\n')
echo "Excluded files: ${excludedFiles}"
+ echo "Excluded files (as glob): ${excludedFiles}"
+ echo "Excluded files (as regex): ${excludedRegexes}"
echo "Changed files: ${changedFiles}"
nonTriggerFiles = changedFiles.every { changed ->
- excludedFiles.any { excluded -> changed ==~ excluded }
+ excludedRegexes.any { regex -> changed ==~ regex }
+ }
+
+ // Log the result
+ if (nonTriggerFiles) {
+ echo "All changed files are non-trigger files."
+ } else {
+ echo "Some changed files are not in the non-trigger list."
+ }
+ }
}
- }
- }
}
}
stage('Run tests for operator') {
From cac0946a656091e0b4a296b86c90ffd41e3f19ab Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 10:00:40 +0200
Subject: [PATCH 08/44] Implement non-trigger files determination logic in
Jenkinsfile with regex support and logging
---
Jenkinsfile | 100 +++++++++++++++++++++++++++++++---------------------
1 file changed, 60 insertions(+), 40 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 979d1cea7..080e26eac 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -266,10 +266,47 @@ pipeline {
disableConcurrentBuilds(abortPrevious: true)
}
stages {
+ stage('Determine non-trigger files') {
+ steps {
+ script {
+ def changesetFile = "non-trigger-files.txt"
+ if (fileExists(changesetFile)) {
+ def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
+
+ def convertGlobToRegex = { glob ->
+ glob.replace("**", ".*").replace("*", "[^/]*")
+ }
+ def excludedRegexes = excludedFiles.collect { convertGlobToRegex(it) }
+
+ def changedFiles = sh(script: "git diff --name-only origin/main", returnStdout: true).trim().split('\n')
+ echo "Excluded files: ${excludedFiles}"
+ echo "Excluded files (as glob): ${excludedFiles}"
+ echo "Excluded files (as regex): ${excludedRegexes}"
+ echo "Changed files: ${changedFiles}"
+
+ nonTriggerFiles = changedFiles.every { changed ->
+ excludedRegexes.any { regex -> changed ==~ regex }
+ }
+
+ // Log the result
+ if (nonTriggerFiles) {
+ echo "All changed files are non-trigger files."
+ } else {
+ echo "Some changed files are not in the non-trigger list."
+ }
+ }
+ }
+ }
+ }
stage('Prepare') {
when {
- expression {
- !skipBranchBuilds
+ allOf {
+ expression {
+ !nonTriggerFiles
+ }
+ expression {
+ !skipBranchBuilds
+ }
}
}
steps {
@@ -323,8 +360,13 @@ EOF
}
stage('Build docker image') {
when {
- expression {
- !skipBranchBuilds
+ allOf {
+ expression {
+ !nonTriggerFiles
+ }
+ expression {
+ !skipBranchBuilds
+ }
}
}
steps {
@@ -350,8 +392,13 @@ EOF
}
stage('GoLicenseDetector test') {
when {
- expression {
- !skipBranchBuilds
+ allOf {
+ expression {
+ !nonTriggerFiles
+ }
+ expression {
+ !skipBranchBuilds
+ }
}
}
steps {
@@ -377,8 +424,13 @@ EOF
}
stage('GoLicense test') {
when {
- expression {
- !skipBranchBuilds
+ allOf {
+ expression {
+ !nonTriggerFiles
+ }
+ expression {
+ !skipBranchBuilds
+ }
}
}
steps {
@@ -409,38 +461,6 @@ EOF
}
}
}
- stage('Determine non-trigger files') {
- steps {
- script {
- def changesetFile = "non-trigger-files.txt"
- if (fileExists(changesetFile)) {
- def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
-
- def convertGlobToRegex = { glob ->
- glob.replace("**", ".*").replace("*", "[^/]*")
- }
- def excludedRegexes = excludedFiles.collect { convertGlobToRegex(it) }
-
- def changedFiles = sh(script: "git diff --name-only origin/main", returnStdout: true).trim().split('\n')
- echo "Excluded files: ${excludedFiles}"
- echo "Excluded files (as glob): ${excludedFiles}"
- echo "Excluded files (as regex): ${excludedRegexes}"
- echo "Changed files: ${changedFiles}"
-
- nonTriggerFiles = changedFiles.every { changed ->
- excludedRegexes.any { regex -> changed ==~ regex }
- }
-
- // Log the result
- if (nonTriggerFiles) {
- echo "All changed files are non-trigger files."
- } else {
- echo "Some changed files are not in the non-trigger list."
- }
- }
- }
- }
- }
stage('Run tests for operator') {
when {
allOf {
From 2c46d1f046fbc8691840be50044fd30f97ab6a6c Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 10:27:03 +0200
Subject: [PATCH 09/44] Replace non-trigger-files.txt with .e2eignore for file
exclusion in Jenkinsfile
---
non-trigger-files.txt => .e2eignore | 3 ++-
Jenkinsfile | 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
rename non-trigger-files.txt => .e2eignore (70%)
diff --git a/non-trigger-files.txt b/.e2eignore
similarity index 70%
rename from non-trigger-files.txt
rename to .e2eignore
index 2600d5815..de023e262 100644
--- a/non-trigger-files.txt
+++ b/.e2eignore
@@ -3,4 +3,5 @@ LICENSE
operator.png
README.md
kubernetes.svg
-non-trigger-files.txt
+.e2eignore
+Jenkinsfile
diff --git a/Jenkinsfile b/Jenkinsfile
index 080e26eac..0ff3a26e1 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -269,7 +269,7 @@ pipeline {
stage('Determine non-trigger files') {
steps {
script {
- def changesetFile = "non-trigger-files.txt"
+ def changesetFile = ".e2eignore"
if (fileExists(changesetFile)) {
def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
@@ -294,8 +294,8 @@ pipeline {
} else {
echo "Some changed files are not in the non-trigger list."
}
- }
- }
+ }
+ }
}
}
stage('Prepare') {
From 0087f392b18564d38cf9d367bf13e2d3e9dff758 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 12:00:56 +0200
Subject: [PATCH 10/44] Add checkE2EIgnoreFiles function to validate changed
files against .e2eignore
---
Jenkinsfile | 107 +++++++++++++++++-----------------------------------
1 file changed, 35 insertions(+), 72 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 0ff3a26e1..066c42f6e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -171,6 +171,10 @@ void makeReport() {
}
TestsReport = TestsReport + "\r\n| We run $startedTestAmount out of $wholeTestAmount|"
TestsReportXML = TestsReportXML + '\n'
+
+ sh """
+ echo "${TestsReportXML}" > TestsReport.xml
+ """
}
void clusterRunner(String cluster) {
@@ -241,13 +245,31 @@ void runTest(Integer TEST_ID) {
}
}
+void checkE2EIgnoreFiles() {
+ def e2eignoreFile = ".e2eignore"
+ if (fileExists(e2eignoreFile)) {
+ def excludedFiles = readFile(e2eignoreFile).split('\n').collect {it.trim()}
+ def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
+ echo "Excluded files: ${excludedFiles}"
+ echo "Changed files: ${changedFiles}"
+
+ def excludedFilesRegex = excludedFiles.collect {it.replace("**", ".*").replace("*", "[^/]*")}
+ def excludedFileList = changedFiles.every {changed -> excludedFilesRegex.any { regex -> changed ==~ regex}}
+
+ if (excludedFileList) {
+ currentBuild.result = 'ABORTED'
+ error("All changed files are e2eignore files. Aborting pipeline execution.")
+ } else {
+ echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
+ }
+ }
+}
+
def skipBranchBuilds = true
if (env.CHANGE_URL) {
skipBranchBuilds = false
}
-def nonTriggerFiles = false
-
pipeline {
environment {
CLOUDSDK_CORE_DISABLE_PROMPTS = 1
@@ -266,50 +288,14 @@ pipeline {
disableConcurrentBuilds(abortPrevious: true)
}
stages {
- stage('Determine non-trigger files') {
- steps {
- script {
- def changesetFile = ".e2eignore"
- if (fileExists(changesetFile)) {
- def excludedFiles = readFile(changesetFile).split('\n').collect {it.trim()}
-
- def convertGlobToRegex = { glob ->
- glob.replace("**", ".*").replace("*", "[^/]*")
- }
- def excludedRegexes = excludedFiles.collect { convertGlobToRegex(it) }
-
- def changedFiles = sh(script: "git diff --name-only origin/main", returnStdout: true).trim().split('\n')
- echo "Excluded files: ${excludedFiles}"
- echo "Excluded files (as glob): ${excludedFiles}"
- echo "Excluded files (as regex): ${excludedRegexes}"
- echo "Changed files: ${changedFiles}"
-
- nonTriggerFiles = changedFiles.every { changed ->
- excludedRegexes.any { regex -> changed ==~ regex }
- }
-
- // Log the result
- if (nonTriggerFiles) {
- echo "All changed files are non-trigger files."
- } else {
- echo "Some changed files are not in the non-trigger list."
- }
- }
- }
- }
- }
stage('Prepare') {
when {
- allOf {
- expression {
- !nonTriggerFiles
- }
- expression {
- !skipBranchBuilds
- }
+ expression {
+ !skipBranchBuilds
}
}
steps {
+ checkE2EIgnoreFiles()
initTests()
script {
if (AUTHOR_NAME == 'null') {
@@ -360,13 +346,8 @@ EOF
}
stage('Build docker image') {
when {
- allOf {
- expression {
- !nonTriggerFiles
- }
- expression {
- !skipBranchBuilds
- }
+ expression {
+ !skipBranchBuilds
}
}
steps {
@@ -392,13 +373,8 @@ EOF
}
stage('GoLicenseDetector test') {
when {
- allOf {
- expression {
- !nonTriggerFiles
- }
- expression {
- !skipBranchBuilds
- }
+ expression {
+ !skipBranchBuilds
}
}
steps {
@@ -424,13 +400,8 @@ EOF
}
stage('GoLicense test') {
when {
- allOf {
- expression {
- !nonTriggerFiles
- }
- expression {
- !skipBranchBuilds
- }
+ expression {
+ !skipBranchBuilds
}
}
steps {
@@ -463,13 +434,8 @@ EOF
}
stage('Run tests for operator') {
when {
- allOf {
- expression {
- !nonTriggerFiles
- }
- expression {
- !skipBranchBuilds
- }
+ expression {
+ !skipBranchBuilds
}
}
options {
@@ -547,9 +513,6 @@ EOF
}
}
makeReport()
- sh """
- echo "${TestsReportXML}" > TestsReport.xml
- """
step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
archiveArtifacts '*.xml'
From 5f6f92cb506d95ce4d64bb0bc4cfef39a0a056d8 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 12:03:39 +0200
Subject: [PATCH 11/44] Remove .e2eignore and Jenkinsfile from exclusion list
in .e2eignore
---
.e2eignore | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/.e2eignore b/.e2eignore
index de023e262..d65fdd128 100644
--- a/.e2eignore
+++ b/.e2eignore
@@ -3,5 +3,4 @@ LICENSE
operator.png
README.md
kubernetes.svg
-.e2eignore
-Jenkinsfile
+.e2eignore
\ No newline at end of file
From ff4ec1ae635c1cc3d3cd55f23707f018b4081d32 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 12:09:54 +0200
Subject: [PATCH 12/44] improved readability and consistency
---
Jenkinsfile | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 066c42f6e..bd609c1e7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -248,13 +248,13 @@ void runTest(Integer TEST_ID) {
void checkE2EIgnoreFiles() {
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
- def excludedFiles = readFile(e2eignoreFile).split('\n').collect {it.trim()}
+ def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
echo "Excluded files: ${excludedFiles}"
echo "Changed files: ${changedFiles}"
- def excludedFilesRegex = excludedFiles.collect {it.replace("**", ".*").replace("*", "[^/]*")}
- def excludedFileList = changedFiles.every {changed -> excludedFilesRegex.any { regex -> changed ==~ regex}}
+ def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
+ def excludedFileList = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
if (excludedFileList) {
currentBuild.result = 'ABORTED'
From 85a318776ecfbd67b7eb47bbb6140b89aa528c73 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 13:42:11 +0200
Subject: [PATCH 13/44] Refactor Jenkinsfile to improve handling of e2eignore
files and streamline pipeline execution logic
---
Jenkinsfile | 75 +++++++++++++++++++++++++++--------------------------
1 file changed, 38 insertions(+), 37 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index bd609c1e7..9744d0d01 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -254,11 +254,10 @@ void checkE2EIgnoreFiles() {
echo "Changed files: ${changedFiles}"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
- def excludedFileList = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
+ def onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
- if (excludedFileList) {
- currentBuild.result = 'ABORTED'
- error("All changed files are e2eignore files. Aborting pipeline execution.")
+ if (onlyIgnoredFiles) {
+ echo "All changed files are e2eignore files. Aborting pipeline execution."
} else {
echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
}
@@ -291,7 +290,7 @@ pipeline {
stage('Prepare') {
when {
expression {
- !skipBranchBuilds
+ !skipBranchBuilds && !onlyIgnoredFiles
}
}
steps {
@@ -347,7 +346,7 @@ EOF
stage('Build docker image') {
when {
expression {
- !skipBranchBuilds
+ !skipBranchBuilds && !onlyIgnoredFiles
}
}
steps {
@@ -374,7 +373,7 @@ EOF
stage('GoLicenseDetector test') {
when {
expression {
- !skipBranchBuilds
+ !skipBranchBuilds && !onlyIgnoredFiles
}
}
steps {
@@ -401,7 +400,7 @@ EOF
stage('GoLicense test') {
when {
expression {
- !skipBranchBuilds
+ !skipBranchBuilds && !onlyIgnoredFiles
}
}
steps {
@@ -435,7 +434,7 @@ EOF
stage('Run tests for operator') {
when {
expression {
- !skipBranchBuilds
+ !skipBranchBuilds && !onlyIgnoredFiles
}
}
options {
@@ -493,41 +492,43 @@ EOF
post {
always {
script {
- echo "CLUSTER ASSIGNMENTS\n" + tests.toString().replace("], ","]\n").replace("]]","]").replaceFirst("\\[","")
+ if (!onlyIgnoredFiles) {
+ echo "CLUSTER ASSIGNMENTS\n" + tests.toString().replace("], ","]\n").replace("]]","]").replaceFirst("\\[","")
- if (currentBuild.result != null && currentBuild.result != 'SUCCESS' && currentBuild.nextBuild == null) {
- try {
- slackSend channel: "@${AUTHOR_NAME}", color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
- }
- catch (exc) {
- slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
+ if (currentBuild.result != null && currentBuild.result != 'SUCCESS' && currentBuild.nextBuild == null) {
+ try {
+ slackSend channel: "@${AUTHOR_NAME}", color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
+ }
+ catch (exc) {
+ slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
+ }
}
- }
- if (env.CHANGE_URL && currentBuild.nextBuild == null) {
- for (comment in pullRequest.comments) {
- println("Author: ${comment.user}, Comment: ${comment.body}")
- if (comment.user.equals('JNKPercona')) {
- println("delete comment")
- comment.delete()
+ if (env.CHANGE_URL && currentBuild.nextBuild == null) {
+ for (comment in pullRequest.comments) {
+ println("Author: ${comment.user}, Comment: ${comment.body}")
+ if (comment.user.equals('JNKPercona')) {
+ println("delete comment")
+ comment.delete()
+ }
}
+ makeReport()
+ step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
+ archiveArtifacts '*.xml'
+
+ unstash 'IMAGE'
+ def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
+ TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
+ pullRequest.comment(TestsReport)
}
- makeReport()
- step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
- archiveArtifacts '*.xml'
-
- unstash 'IMAGE'
- def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
- TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
- pullRequest.comment(TestsReport)
+ deleteOldClusters("$CLUSTER_NAME")
+ sh """
+ sudo docker system prune --volumes -af
+ sudo rm -rf *
+ """
+ deleteDir()
}
}
- deleteOldClusters("$CLUSTER_NAME")
- sh """
- sudo docker system prune --volumes -af
- sudo rm -rf *
- """
- deleteDir()
}
}
}
From 3498d9897a4ef89b9a42b8bbffc22bf87446f2b5 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 13:54:16 +0200
Subject: [PATCH 14/44] Add onlyIgnoredFiles flag to Jenkinsfile for enhanced
e2eignore handling
---
Jenkinsfile | 1 +
1 file changed, 1 insertion(+)
diff --git a/Jenkinsfile b/Jenkinsfile
index 9744d0d01..87e185d17 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -245,6 +245,7 @@ void runTest(Integer TEST_ID) {
}
}
+def onlyIgnoredFiles = false
void checkE2EIgnoreFiles() {
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
From 60c267b1a4a89d06a6ccf564d62af40a2fc985f1 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 13:54:28 +0200
Subject: [PATCH 15/44] Fix variable declaration for onlyIgnoredFiles in
checkE2EIgnoreFiles function
---
Jenkinsfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 87e185d17..ef100498f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -255,7 +255,7 @@ void checkE2EIgnoreFiles() {
echo "Changed files: ${changedFiles}"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
- def onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
+ onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
if (onlyIgnoredFiles) {
echo "All changed files are e2eignore files. Aborting pipeline execution."
From 34bcf49eff652d7049648e5efeb8dc0a7f36f976 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 14:03:55 +0200
Subject: [PATCH 16/44] testing the new behavior of skipping builds for
non-trigger files
---
README.md | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/README.md b/README.md
index fbcaec126..aca751c74 100644
--- a/README.md
+++ b/README.md
@@ -26,7 +26,7 @@ You interact with Percona Operator mostly via the command line tool. If you feel
# Architecture
-Percona Operators are based on the [Operator SDK](https://github.com/operator-framework/operator-sdk) and leverage Kubernetes primitives to follow best CNCF practices.
+Percona Operators are based on the [Operator SDK](https://github.com/operator-framework/operator-sdk) and leverage Kubernetes primitives to follow best CNCF practices.
Please read more about [architecture and design decisions](https://www.percona.com/doc/kubernetes-operator-for-pxc/architecture.html).
@@ -36,7 +36,7 @@ To learn more about the Operator, check the [Percona Operator for MySQL based on
# Quickstart installation
-Ready to try out the Operator? Check the [Quickstart tutorial](https://docs.percona.com/percona-operator-for-mysql/pxc/quickstart.html) for easy-to follow steps.
+Ready to try out the Operator? Check the [Quickstart tutorial](https://docs.percona.com/percona-operator-for-mysql/pxc/quickstart.html) for easy-to follow steps.
Below is one of the ways to deploy the Operator using `kubectl`.
@@ -67,22 +67,22 @@ See the [Contribution Guide](CONTRIBUTING.md) and [Building and Testing Guide](e
We would love to hear from you! Reach out to us on [Forum](https://forums.percona.com/c/mysql-mariadb/percona-kubernetes-operator-for-mysql/28) with your questions, feedback and ideas
-# Join Percona Kubernetes Squad!
-```
- % _____
- %%% | __ \
- ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _
- ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` |
- #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| |
- ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_|
+# Join Percona Kubernetes Squad!
+```
+ % _____
+ %%% | __ \
+ ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _
+ ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` |
+ #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| |
+ ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_|
,((### ### %%% _ _ _____ _
- (((( (### #### %%%% | | / _ \ / ____| | |
- ((( ((# ###### | | _| (_) |___ | (___ __ _ _ _ __ _ __| |
+ (((( (### #### %%%% | | / _ \ / ____| | |
+ ((( ((# ###### | | _| (_) |___ | (___ __ _ _ _ __ _ __| |
(((( (((# #### | |/ /> _
Date: Fri, 6 Dec 2024 14:11:53 +0200
Subject: [PATCH 17/44] Refactor Jenkinsfile to enhance build logic and
streamline post-build actions
---
Jenkinsfile | 58 ++++++++++++++++++++++++++---------------------------
1 file changed, 28 insertions(+), 30 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index ef100498f..ad0a28016 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -493,42 +493,40 @@ EOF
post {
always {
script {
- if (!onlyIgnoredFiles) {
- echo "CLUSTER ASSIGNMENTS\n" + tests.toString().replace("], ","]\n").replace("]]","]").replaceFirst("\\[","")
+ echo "CLUSTER ASSIGNMENTS\n" + tests.toString().replace("], ","]\n").replace("]]","]").replaceFirst("\\[","")
- if (currentBuild.result != null && currentBuild.result != 'SUCCESS' && currentBuild.nextBuild == null) {
- try {
- slackSend channel: "@${AUTHOR_NAME}", color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
- }
- catch (exc) {
- slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
- }
+ if (currentBuild.result != null && currentBuild.result != 'SUCCESS' && currentBuild.nextBuild == null) {
+ try {
+ slackSend channel: "@${AUTHOR_NAME}", color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
}
+ catch (exc) {
+ slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
+ }
+ }
- if (env.CHANGE_URL && currentBuild.nextBuild == null) {
- for (comment in pullRequest.comments) {
- println("Author: ${comment.user}, Comment: ${comment.body}")
- if (comment.user.equals('JNKPercona')) {
- println("delete comment")
- comment.delete()
- }
+ if (!skipBranchBuilds && !onlyIgnoredFiles && currentBuild.nextBuild == null) {
+ for (comment in pullRequest.comments) {
+ println("Author: ${comment.user}, Comment: ${comment.body}")
+ if (comment.user.equals('JNKPercona')) {
+ println("delete comment")
+ comment.delete()
}
- makeReport()
- step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
- archiveArtifacts '*.xml'
-
- unstash 'IMAGE'
- def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
- TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
- pullRequest.comment(TestsReport)
}
- deleteOldClusters("$CLUSTER_NAME")
- sh """
- sudo docker system prune --volumes -af
- sudo rm -rf *
- """
- deleteDir()
+ makeReport()
+ step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
+ archiveArtifacts '*.xml'
+
+ unstash 'IMAGE'
+ def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
+ TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
+ pullRequest.comment(TestsReport)
}
+ deleteOldClusters("$CLUSTER_NAME")
+ sh """
+ sudo docker system prune --volumes -af
+ sudo rm -rf *
+ """
+ deleteDir()
}
}
}
From ded65b43bd10375e89c965227e8a79b80622d5bb Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 22:50:32 +0200
Subject: [PATCH 18/44] Update .e2eignore to include additional documentation
files
---
.e2eignore | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/.e2eignore b/.e2eignore
index d65fdd128..b1563faf1 100644
--- a/.e2eignore
+++ b/.e2eignore
@@ -1,6 +1,10 @@
docs/**
+code-of-conduct.md
+CONTRIBUTING.md
+README.md
+.gitattributes
+.gitignore
LICENSE
operator.png
-README.md
kubernetes.svg
.e2eignore
\ No newline at end of file
From 16195034cbc2af173ec8f48f29bbf50a2db4b508 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 6 Dec 2024 23:32:49 +0200
Subject: [PATCH 19/44] debug
---
.e2eignore | 3 ++-
Jenkinsfile | 11 ++++++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/.e2eignore b/.e2eignore
index b1563faf1..d88a65fb8 100644
--- a/.e2eignore
+++ b/.e2eignore
@@ -7,4 +7,5 @@ README.md
LICENSE
operator.png
kubernetes.svg
-.e2eignore
\ No newline at end of file
+.e2eignore
+Jenkinsfile
diff --git a/Jenkinsfile b/Jenkinsfile
index ad0a28016..6cade48b4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -250,10 +250,19 @@ void checkE2EIgnoreFiles() {
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
- def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
+
+ // def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
+
+ // Find the last merge commit (sync)
+ def mergeCommitSHA = sh(script: "git log --oneline --grep='Merge branch' -n 1 --format='%H'", returnStdout: true).trim()
+ // Get files changed since that merge commit
+ def changedFiles = sh(script: "git diff --name-only ${mergeCommitSHA}..HEAD", returnStdout: true).trim().split('\n')
+
echo "Excluded files: ${excludedFiles}"
echo "Changed files: ${changedFiles}"
+
+
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
From 0804924010ccdc0c456ac6252e1b7a95c1f32974 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 01:04:14 +0200
Subject: [PATCH 20/44] avoiding unnecessary test executions for subsequent
commits
---
Jenkinsfile | 36 +++++++++++++++++++++++++++---------
1 file changed, 27 insertions(+), 9 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6cade48b4..2f73a7d10 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -251,17 +251,30 @@ void checkE2EIgnoreFiles() {
if (fileExists(e2eignoreFile)) {
def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
- // def changedFiles = sh(script: "git diff --name-only origin/${env.CHANGE_TARGET}", returnStdout: true).trim().split('\n')
-
- // Find the last merge commit (sync)
- def mergeCommitSHA = sh(script: "git log --oneline --grep='Merge branch' -n 1 --format='%H'", returnStdout: true).trim()
- // Get files changed since that merge commit
- def changedFiles = sh(script: "git diff --name-only ${mergeCommitSHA}..HEAD", returnStdout: true).trim().split('\n')
-
- echo "Excluded files: ${excludedFiles}"
- echo "Changed files: ${changedFiles}"
+ def lastProcessedCommitFile="last-processed-commit.txt"
+ def lastProcessedCommit = ""
+ def previousBuild = currentBuild.previousBuild
+ if (previousBuild != null && previousBuild.result == 'SUCCESS') {
+ try {
+ previousBuild.copyArtifact(${lastProcessedCommitFile})
+ lastProcessedCommit = readFile(${lastProcessedCommitFile}).trim()
+ } catch (Exception e) {
+ echo "No ${lastProcessedCommitFile} file found from previous build. Assuming this is the first run."
+ }
+ } else {
+ echo "No previous successful build found."
+ }
+ if (lastProcessedCommit == "") {
+ echo "This is the first run. Using merge base as the starting point for the diff."
+ def changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n')
+ } else {
+ echo "Processing changes since last processed commit: $lastProcessedCommit"
+ def changedFiles = sh(script: "git diff --name-only $lastProcessedCommit HEAD", returnStdout: true).trim().split('\n')
+ }
+ echo "Excluded files: $excludedFiles"
+ echo "Changed files: $changedFiles"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
@@ -271,6 +284,11 @@ void checkE2EIgnoreFiles() {
} else {
echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
}
+
+ sh """
+ echo \$(git rev-parse HEAD) > ${lastProcessedCommitFile}
+ """
+ archiveArtifacts ${lastProcessedCommitFile}
}
}
From 47cd5e5b2d186071e0abbc10e7d023a06bce4908 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 01:38:36 +0200
Subject: [PATCH 21/44] debug
---
Jenkinsfile | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 2f73a7d10..6c0793932 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -265,12 +265,13 @@ void checkE2EIgnoreFiles() {
echo "No previous successful build found."
}
+ def changedFiles = []
if (lastProcessedCommit == "") {
echo "This is the first run. Using merge base as the starting point for the diff."
- def changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n')
+ changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n')
} else {
echo "Processing changes since last processed commit: $lastProcessedCommit"
- def changedFiles = sh(script: "git diff --name-only $lastProcessedCommit HEAD", returnStdout: true).trim().split('\n')
+ changedFiles = sh(script: "git diff --name-only $lastProcessedCommit HEAD", returnStdout: true).trim().split('\n')
}
echo "Excluded files: $excludedFiles"
@@ -279,6 +280,7 @@ void checkE2EIgnoreFiles() {
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
+ echo "onlyIgnoredFiles: $onlyIgnoredFiles"
if (onlyIgnoredFiles) {
echo "All changed files are e2eignore files. Aborting pipeline execution."
} else {
@@ -315,6 +317,11 @@ pipeline {
disableConcurrentBuilds(abortPrevious: true)
}
stages {
+ stage('Check Ignore Files') {
+ steps {
+ checkE2EIgnoreFiles()
+ }
+ }
stage('Prepare') {
when {
expression {
@@ -322,7 +329,6 @@ pipeline {
}
}
steps {
- checkE2EIgnoreFiles()
initTests()
script {
if (AUTHOR_NAME == 'null') {
@@ -530,7 +536,9 @@ EOF
slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
}
}
-
+ echo "skipBranchBuilds: ${skipBranchBuilds}"
+ echo "onlyIgnoredFiles: ${onlyIgnoredFiles}"
+ echo "currentBuild.nextBuild: ${currentBuild.nextBuild}"
if (!skipBranchBuilds && !onlyIgnoredFiles && currentBuild.nextBuild == null) {
for (comment in pullRequest.comments) {
println("Author: ${comment.user}, Comment: ${comment.body}")
From 344ce42c4d6d56c5bf1da921c5f52c69e911c6d0 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 10:33:05 +0200
Subject: [PATCH 22/44] Refactor Jenkinsfile to improve variable usage and
enhance readability
---
Jenkinsfile | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6c0793932..71deee83e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -245,7 +245,7 @@ void runTest(Integer TEST_ID) {
}
}
-def onlyIgnoredFiles = false
+onlyIgnoredFiles = false
void checkE2EIgnoreFiles() {
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
@@ -256,10 +256,10 @@ void checkE2EIgnoreFiles() {
def previousBuild = currentBuild.previousBuild
if (previousBuild != null && previousBuild.result == 'SUCCESS') {
try {
- previousBuild.copyArtifact(${lastProcessedCommitFile})
- lastProcessedCommit = readFile(${lastProcessedCommitFile}).trim()
+ previousBuild.copyArtifact($lastProcessedCommitFile)
+ lastProcessedCommit = readFile($lastProcessedCommitFile).trim()
} catch (Exception e) {
- echo "No ${lastProcessedCommitFile} file found from previous build. Assuming this is the first run."
+ echo "No $lastProcessedCommitFile file found from previous build. Assuming this is the first run."
}
} else {
echo "No previous successful build found."
@@ -288,9 +288,11 @@ void checkE2EIgnoreFiles() {
}
sh """
- echo \$(git rev-parse HEAD) > ${lastProcessedCommitFile}
+ echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
"""
- archiveArtifacts ${lastProcessedCommitFile}
+ archiveArtifacts "$lastProcessedCommitFile"
+
+ onlyIgnoredFiles = true
}
}
From fff22a74435221f1888188ff76cdc3e11e3856ed Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 10:34:04 +0200
Subject: [PATCH 23/44] Remove unnecessary debug echo statements from
Jenkinsfile to streamline output
---
Jenkinsfile | 6 ------
1 file changed, 6 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 71deee83e..1e273e94a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -280,7 +280,6 @@ void checkE2EIgnoreFiles() {
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
- echo "onlyIgnoredFiles: $onlyIgnoredFiles"
if (onlyIgnoredFiles) {
echo "All changed files are e2eignore files. Aborting pipeline execution."
} else {
@@ -291,8 +290,6 @@ void checkE2EIgnoreFiles() {
echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
"""
archiveArtifacts "$lastProcessedCommitFile"
-
- onlyIgnoredFiles = true
}
}
@@ -538,9 +535,6 @@ EOF
slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
}
}
- echo "skipBranchBuilds: ${skipBranchBuilds}"
- echo "onlyIgnoredFiles: ${onlyIgnoredFiles}"
- echo "currentBuild.nextBuild: ${currentBuild.nextBuild}"
if (!skipBranchBuilds && !onlyIgnoredFiles && currentBuild.nextBuild == null) {
for (comment in pullRequest.comments) {
println("Author: ${comment.user}, Comment: ${comment.body}")
From 12cc48f7ef366a21da87019d475b2041f0a8930f Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 10:42:53 +0200
Subject: [PATCH 24/44] Remove unused variable 'changedFiles' from Jenkinsfile
to clean up code
---
Jenkinsfile | 1 -
1 file changed, 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 1e273e94a..a445d573f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -265,7 +265,6 @@ void checkE2EIgnoreFiles() {
echo "No previous successful build found."
}
- def changedFiles = []
if (lastProcessedCommit == "") {
echo "This is the first run. Using merge base as the starting point for the diff."
changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n')
From c23fb588fe03ff1703cc1ed374dbccd37b63505a Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 11:42:48 +0200
Subject: [PATCH 25/44] debug
---
Jenkinsfile | 72 ++++++++++++++++++++++++++++++++++-------------------
1 file changed, 47 insertions(+), 25 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index a445d573f..acfed0581 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -254,10 +254,21 @@ void checkE2EIgnoreFiles() {
def lastProcessedCommitFile="last-processed-commit.txt"
def lastProcessedCommit = ""
def previousBuild = currentBuild.previousBuild
+ echo "previousBuild: $previousBuild"
if (previousBuild != null && previousBuild.result == 'SUCCESS') {
try {
- previousBuild.copyArtifact($lastProcessedCommitFile)
- lastProcessedCommit = readFile($lastProcessedCommitFile).trim()
+ // previousBuild.copyArtifact("$lastProcessedCommitFile")
+
+ copyArtifacts(
+ projectName: env.JOB_NAME, // Name of the current job
+ selector: specific("${previousBuild.number}"), // Reference the previous build by number
+ filter: "$lastProcessedCommitFile", // Specify the file(s) to copy
+ flatten: true // Optional: to avoid recreating directory structure
+ )
+
+
+ lastProcessedCommit = readFile("$lastProcessedCommitFile").trim()
+ echo "lastProcessedCommit: $lastProcessedCommit"
} catch (Exception e) {
echo "No $lastProcessedCommitFile file found from previous build. Assuming this is the first run."
}
@@ -277,14 +288,22 @@ void checkE2EIgnoreFiles() {
echo "Changed files: $changedFiles"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
- onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
- if (onlyIgnoredFiles) {
- echo "All changed files are e2eignore files. Aborting pipeline execution."
+ if (changedFiles.toList().isEmpty()) {
+ onlyIgnoredFiles = true
+ echo "No files were changed. Aborting pipeline execution."
} else {
- echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
+ onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
+
+ if (onlyIgnoredFiles) {
+ echo "All changed files are e2eignore files. Aborting pipeline execution."
+ } else {
+ echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
+ }
}
+ echo "onlyIgnoredFiles: $onlyIgnoredFiles"
+
sh """
echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
"""
@@ -313,6 +332,7 @@ pipeline {
}
options {
disableConcurrentBuilds(abortPrevious: true)
+ copyArtifactPermission("$JOB_NAME/PR-*")
}
stages {
stage('Check Ignore Files') {
@@ -534,28 +554,30 @@ EOF
slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
}
}
- if (!skipBranchBuilds && !onlyIgnoredFiles && currentBuild.nextBuild == null) {
- for (comment in pullRequest.comments) {
- println("Author: ${comment.user}, Comment: ${comment.body}")
- if (comment.user.equals('JNKPercona')) {
- println("delete comment")
- comment.delete()
+ echo "onlyIgnoredFiles: $onlyIgnoredFiles"
+ if (!onlyIgnoredFiles) {
+ if (!skipBranchBuilds && currentBuild.nextBuild == null) {
+ for (comment in pullRequest.comments) {
+ println("Author: ${comment.user}, Comment: ${comment.body}")
+ if (comment.user.equals('JNKPercona')) {
+ println("delete comment")
+ comment.delete()
+ }
}
+ makeReport()
+ step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
+ archiveArtifacts '*.xml'
+
+ unstash 'IMAGE'
+ def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
+ TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
+ pullRequest.comment(TestsReport)
}
- makeReport()
- step([$class: 'JUnitResultArchiver', testResults: '*.xml', healthScaleFactor: 1.0])
- archiveArtifacts '*.xml'
-
- unstash 'IMAGE'
- def IMAGE = sh(returnStdout: true, script: "cat results/docker/TAG").trim()
- TestsReport = TestsReport + "\r\n\r\ncommit: ${env.CHANGE_URL}/commits/${env.GIT_COMMIT}\r\nimage: `${IMAGE}`\r\n"
- pullRequest.comment(TestsReport)
+ deleteOldClusters("$CLUSTER_NAME")
+ sh """
+ sudo docker system prune --volumes -af
+ """
}
- deleteOldClusters("$CLUSTER_NAME")
- sh """
- sudo docker system prune --volumes -af
- sudo rm -rf *
- """
deleteDir()
}
}
From 78f5373e42a254a23957ffc83d8ed90b747194d4 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 12:40:20 +0200
Subject: [PATCH 26/44] Refactor Jenkinsfile to enhance previous build handling
and streamline commit processing logic
---
Jenkinsfile | 48 ++++++++++++++++++++++++++----------------------
1 file changed, 26 insertions(+), 22 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index acfed0581..7331a5be6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -253,35 +253,39 @@ void checkE2EIgnoreFiles() {
def lastProcessedCommitFile="last-processed-commit.txt"
def lastProcessedCommit = ""
- def previousBuild = currentBuild.previousBuild
- echo "previousBuild: $previousBuild"
- if (previousBuild != null && previousBuild.result == 'SUCCESS') {
- try {
- // previousBuild.copyArtifact("$lastProcessedCommitFile")
-
- copyArtifacts(
- projectName: env.JOB_NAME, // Name of the current job
- selector: specific("${previousBuild.number}"), // Reference the previous build by number
- filter: "$lastProcessedCommitFile", // Specify the file(s) to copy
- flatten: true // Optional: to avoid recreating directory structure
- )
-
-
- lastProcessedCommit = readFile("$lastProcessedCommitFile").trim()
- echo "lastProcessedCommit: $lastProcessedCommit"
- } catch (Exception e) {
- echo "No $lastProcessedCommitFile file found from previous build. Assuming this is the first run."
+
+
+ def currentBuildRef = currentBuild.previousBuild
+ while (currentBuildRef != null) {
+ if (currentBuildRef.result == 'SUCCESS') {
+ try {
+ echo "Found a previous successful build: ${currentBuildRef.number}"
+
+ copyArtifacts(
+ projectName: env.JOB_NAME, // Current job name
+ selector: specific("${currentBuildRef.number}"), // Specific successful build
+ filter: "$lastProcessedCommitFile", // File to copy
+ flatten: true // Avoid recreating directory structure
+ )
+
+ lastProcessedCommit = readFile("$lastProcessedCommitFile").trim()
+ echo "lastProcessedCommit: $lastProcessedCommit"
+ break
+ } catch (Exception e) {
+ echo "No $lastProcessedCommitFile found in build ${currentBuildRef.number}. Checking earlier builds."
+ }
+ } else {
+ echo "Build ${currentBuildRef.number} was not successful. Checking earlier builds."
}
- } else {
- echo "No previous successful build found."
+ currentBuildRef = currentBuildRef.previousBuild
}
if (lastProcessedCommit == "") {
echo "This is the first run. Using merge base as the starting point for the diff."
- changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n')
+ changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n').findAll{it}
} else {
echo "Processing changes since last processed commit: $lastProcessedCommit"
- changedFiles = sh(script: "git diff --name-only $lastProcessedCommit HEAD", returnStdout: true).trim().split('\n')
+ changedFiles = sh(script: "git diff --name-only $lastProcessedCommit HEAD", returnStdout: true).trim().split('\n').findAll{it}
}
echo "Excluded files: $excludedFiles"
From 11532fed9c71873a0a4eb6f3ec38c541dae6da4f Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 12:44:12 +0200
Subject: [PATCH 27/44] Remove redundant echo statements for 'onlyIgnoredFiles'
in Jenkinsfile to clean up output
---
Jenkinsfile | 3 ---
1 file changed, 3 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 7331a5be6..452e70830 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -306,8 +306,6 @@ void checkE2EIgnoreFiles() {
}
}
- echo "onlyIgnoredFiles: $onlyIgnoredFiles"
-
sh """
echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
"""
@@ -558,7 +556,6 @@ EOF
slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
}
}
- echo "onlyIgnoredFiles: $onlyIgnoredFiles"
if (!onlyIgnoredFiles) {
if (!skipBranchBuilds && currentBuild.nextBuild == null) {
for (comment in pullRequest.comments) {
From fa716b1c41e50976132cee882cadade2f5319b2d Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 12:48:35 +0200
Subject: [PATCH 28/44] Remove 'Jenkinsfile' from .e2eignore to ensure it is
included in end-to-end tests
---
.e2eignore | 1 -
1 file changed, 1 deletion(-)
diff --git a/.e2eignore b/.e2eignore
index d88a65fb8..6366faa7f 100644
--- a/.e2eignore
+++ b/.e2eignore
@@ -8,4 +8,3 @@ LICENSE
operator.png
kubernetes.svg
.e2eignore
-Jenkinsfile
From b648d4c654d5ef2b049d98f6139591f6e8685bc9 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 12:50:04 +0200
Subject: [PATCH 29/44] Format code in Jenkinsfile for improved readability in
checkE2EIgnoreFiles function
---
Jenkinsfile | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 452e70830..085343c16 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -262,10 +262,10 @@ void checkE2EIgnoreFiles() {
echo "Found a previous successful build: ${currentBuildRef.number}"
copyArtifacts(
- projectName: env.JOB_NAME, // Current job name
- selector: specific("${currentBuildRef.number}"), // Specific successful build
- filter: "$lastProcessedCommitFile", // File to copy
- flatten: true // Avoid recreating directory structure
+ projectName: env.JOB_NAME,
+ selector: specific("${currentBuildRef.number}"),
+ filter: "$lastProcessedCommitFile",
+ flatten: true
)
lastProcessedCommit = readFile("$lastProcessedCommitFile").trim()
From 82d566e2452828339ef549c7686d8b0edb2f7284 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 12:56:06 +0200
Subject: [PATCH 30/44] Refactor checkE2EIgnoreFiles function in Jenkinsfile to
improve variable naming and streamline previous build handling
---
Jenkinsfile | 37 ++++++++++++++-----------------------
1 file changed, 14 insertions(+), 23 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 085343c16..207b04304 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -250,42 +250,33 @@ void checkE2EIgnoreFiles() {
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
-
def lastProcessedCommitFile="last-processed-commit.txt"
- def lastProcessedCommit = ""
-
+ def lastProcessedCommitHash = ""
- def currentBuildRef = currentBuild.previousBuild
- while (currentBuildRef != null) {
- if (currentBuildRef.result == 'SUCCESS') {
+ def build = currentBuild.previousBuild
+ while (build != null) {
+ if (build.result == 'SUCCESS') {
try {
- echo "Found a previous successful build: ${currentBuildRef.number}"
-
- copyArtifacts(
- projectName: env.JOB_NAME,
- selector: specific("${currentBuildRef.number}"),
- filter: "$lastProcessedCommitFile",
- flatten: true
- )
-
- lastProcessedCommit = readFile("$lastProcessedCommitFile").trim()
- echo "lastProcessedCommit: $lastProcessedCommit"
+ echo "Found a previous successful build: ${build.number}"
+ copyArtifacts(projectName: env.JOB_NAME, selector: specific("${build.number}"), filter: "$lastProcessedCommitFile")
+ lastProcessedCommitHash = readFile("$lastProcessedCommitFile").trim()
+ echo "lastProcessedCommitHash: $lastProcessedCommitHash"
break
} catch (Exception e) {
- echo "No $lastProcessedCommitFile found in build ${currentBuildRef.number}. Checking earlier builds."
+ echo "No $lastProcessedCommitFile found in build ${build.number}. Checking earlier builds."
}
} else {
- echo "Build ${currentBuildRef.number} was not successful. Checking earlier builds."
+ echo "Build ${build.number} was not successful. Checking earlier builds."
}
- currentBuildRef = currentBuildRef.previousBuild
+ build = build.previousBuild
}
- if (lastProcessedCommit == "") {
+ if (lastProcessedCommitHash == "") {
echo "This is the first run. Using merge base as the starting point for the diff."
changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n').findAll{it}
} else {
- echo "Processing changes since last processed commit: $lastProcessedCommit"
- changedFiles = sh(script: "git diff --name-only $lastProcessedCommit HEAD", returnStdout: true).trim().split('\n').findAll{it}
+ echo "Processing changes since last processed commit: $lastProcessedCommitHash"
+ changedFiles = sh(script: "git diff --name-only $lastProcessedCommitHash HEAD", returnStdout: true).trim().split('\n').findAll{it}
}
echo "Excluded files: $excludedFiles"
From 0992924672bb9d49054711fa7ab2f293af0d84d2 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 16:03:50 +0200
Subject: [PATCH 31/44] debug to see if the job will run the tests
---
README.md | 1 -
1 file changed, 1 deletion(-)
diff --git a/README.md b/README.md
index aca751c74..5391ee63b 100644
--- a/README.md
+++ b/README.md
@@ -96,4 +96,3 @@ We have a public roadmap which can be found [here](https://github.com/orgs/perco
If you find a bug in Percona Docker Images or in one of the related projects, please submit a report to that project's [JIRA](https://jira.percona.com/browse/K8SPXC) issue tracker or [create a GitHub issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-an-issue#creating-an-issue-from-a-repository) in this repository.
Learn more about submitting bugs, new features ideas and improvements in the [Contribution Guide](CONTRIBUTING.md).
-
From 71534d6eba9ed268669153becc7f47e6ef80b105 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 16:11:59 +0200
Subject: [PATCH 32/44] remove unnecessary if-else statement.
---
Jenkinsfile | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 207b04304..1fabcaf14 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -283,18 +283,12 @@ void checkE2EIgnoreFiles() {
echo "Changed files: $changedFiles"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
+ onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
- if (changedFiles.toList().isEmpty()) {
- onlyIgnoredFiles = true
- echo "No files were changed. Aborting pipeline execution."
+ if (onlyIgnoredFiles) {
+ echo "All changed files are e2eignore files. Aborting pipeline execution."
} else {
- onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
-
- if (onlyIgnoredFiles) {
- echo "All changed files are e2eignore files. Aborting pipeline execution."
- } else {
- echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
- }
+ echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
}
sh """
From cc9921efc3e4ee4a9311ffdba1c5aa647861305b Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 21:12:44 +0200
Subject: [PATCH 33/44] Refactor checkE2EIgnoreFiles function in Jenkinsfile to
improve variable naming and logic for determining test execution
---
Jenkinsfile | 42 ++++++++++++++++++++++++------------------
1 file changed, 24 insertions(+), 18 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 1fabcaf14..9d089c904 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -245,7 +245,7 @@ void runTest(Integer TEST_ID) {
}
}
-onlyIgnoredFiles = false
+needToRunTests = true
void checkE2EIgnoreFiles() {
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
@@ -257,16 +257,16 @@ void checkE2EIgnoreFiles() {
while (build != null) {
if (build.result == 'SUCCESS') {
try {
- echo "Found a previous successful build: ${build.number}"
- copyArtifacts(projectName: env.JOB_NAME, selector: specific("${build.number}"), filter: "$lastProcessedCommitFile")
+ echo "Found a previous successful build: $build.number"
+ copyArtifacts(projectName: env.JOB_NAME, selector: specific("$build.number"), filter: "$lastProcessedCommitFile")
lastProcessedCommitHash = readFile("$lastProcessedCommitFile").trim()
echo "lastProcessedCommitHash: $lastProcessedCommitHash"
break
} catch (Exception e) {
- echo "No $lastProcessedCommitFile found in build ${build.number}. Checking earlier builds."
+ echo "No $lastProcessedCommitFile found in build $build.number. Checking earlier builds."
}
} else {
- echo "Build ${build.number} was not successful. Checking earlier builds."
+ echo "Build $build.number was not successful. Checking earlier builds."
}
build = build.previousBuild
}
@@ -283,12 +283,13 @@ void checkE2EIgnoreFiles() {
echo "Changed files: $changedFiles"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
- onlyIgnoredFiles = changedFiles.every{changed -> excludedFilesRegex.any {regex -> changed ==~ regex}}
+ // needToRunTests = changedFiles.every{changed -> excludedFilesRegex.any{regex -> changed ==~ regex}}
+ needToRunTests = changedFiles.any{changed -> excludedFilesRegex.none{regex -> changed ==~ regex}}
- if (onlyIgnoredFiles) {
- echo "All changed files are e2eignore files. Aborting pipeline execution."
- } else {
+ if (needToRunTests) {
echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
+ } else {
+ echo "All changed files are e2eignore files. Aborting pipeline execution."
}
sh """
@@ -298,9 +299,9 @@ void checkE2EIgnoreFiles() {
}
}
-def skipBranchBuilds = true
+def isPRJob = false
if (env.CHANGE_URL) {
- skipBranchBuilds = false
+ isPRJob = true
}
pipeline {
@@ -323,6 +324,11 @@ pipeline {
}
stages {
stage('Check Ignore Files') {
+ when {
+ expression {
+ isPRJob
+ }
+ }
steps {
checkE2EIgnoreFiles()
}
@@ -330,7 +336,7 @@ pipeline {
stage('Prepare') {
when {
expression {
- !skipBranchBuilds && !onlyIgnoredFiles
+ isPRJob && needToRunTests
}
}
steps {
@@ -385,7 +391,7 @@ EOF
stage('Build docker image') {
when {
expression {
- !skipBranchBuilds && !onlyIgnoredFiles
+ isPRJob && needToRunTests
}
}
steps {
@@ -412,7 +418,7 @@ EOF
stage('GoLicenseDetector test') {
when {
expression {
- !skipBranchBuilds && !onlyIgnoredFiles
+ isPRJob && needToRunTests
}
}
steps {
@@ -439,7 +445,7 @@ EOF
stage('GoLicense test') {
when {
expression {
- !skipBranchBuilds && !onlyIgnoredFiles
+ isPRJob && needToRunTests
}
}
steps {
@@ -473,7 +479,7 @@ EOF
stage('Run tests for operator') {
when {
expression {
- !skipBranchBuilds && !onlyIgnoredFiles
+ isPRJob && needToRunTests
}
}
options {
@@ -541,8 +547,8 @@ EOF
slackSend channel: '#cloud-dev-ci', color: '#FF0000', message: "[${JOB_NAME}]: build ${currentBuild.result}, ${BUILD_URL} owner: @${AUTHOR_NAME}"
}
}
- if (!onlyIgnoredFiles) {
- if (!skipBranchBuilds && currentBuild.nextBuild == null) {
+ if (needToRunTests) {
+ if (isPRJob && currentBuild.nextBuild == null) {
for (comment in pullRequest.comments) {
println("Author: ${comment.user}, Comment: ${comment.body}")
if (comment.user.equals('JNKPercona')) {
From 8c737c28e6909bcf0bcb5ed97961b2c62af0ca48 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 7 Dec 2024 21:21:03 +0200
Subject: [PATCH 34/44] Refactor logic in checkE2EIgnoreFiles function to
simplify test execution determination
---
Jenkinsfile | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 9d089c904..44aab95ab 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -283,8 +283,7 @@ void checkE2EIgnoreFiles() {
echo "Changed files: $changedFiles"
def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
- // needToRunTests = changedFiles.every{changed -> excludedFilesRegex.any{regex -> changed ==~ regex}}
- needToRunTests = changedFiles.any{changed -> excludedFilesRegex.none{regex -> changed ==~ regex}}
+ needToRunTests = !changedFiles.every{changed -> excludedFilesRegex.any{regex -> changed ==~ regex}}
if (needToRunTests) {
echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
From cafffe8966d08c8bc571195b894069c75beef70d Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Mon, 9 Dec 2024 20:32:33 +0200
Subject: [PATCH 35/44] resolving conflicts by accepting the target branch
---
README.md | 38 ++++++++++----------------------------
1 file changed, 10 insertions(+), 28 deletions(-)
diff --git a/README.md b/README.md
index 5391ee63b..186c8c43a 100644
--- a/README.md
+++ b/README.md
@@ -21,8 +21,7 @@ Within the [Percona Operator for MySQL based on Percona XtraDB Cluster](https://
* Automated Password Rotation – use the standard Kubernetes API to enforce password rotation policies for system user
* Private container image registries
-You interact with Percona Operator mostly via the command line tool. If you feel more comfortable with operating the Operator and database clusters via the web interface, there is [Percona Everest](https://docs.percona.com/everest/index.html) - an open-source web-based database provisioning tool available for you. It automates day-to-day database management operations for you, reducing the overall administrative overhead. [Get started with Percona Everest](https://docs.percona.com/everest/quickstart-guide/quick-install.html).
-
+While the Percona Operator is primarily managed through the command line, you can also use **[Percona Everest](https://docs.percona.com/everest/index.html)** for a web-based user interface. This open-source tool provides a streamlined experience for provisioning and managing your databases, simplifying day-to-day tasks and reducing administrative overhead. Learn more about Percona Everest in the [documentation](https://docs.percona.com/everest/index.html) or jump right in with the [quickstart guide](https://docs.percona.com/everest/quickstart-guide/quick-install.html).
# Architecture
@@ -57,41 +56,24 @@ kubectl apply -f https://raw.githubusercontent.com/percona/percona-xtradb-cluste
See full documentation with examples and various advanced cases on [percona.com](https://www.percona.com/doc/kubernetes-operator-for-pxc/index.html).
+# Need help?
+
+**Commercial Support** | **Community Support** |
+:-: | :-: |
+|
Enterprise-grade assistance for your mission-critical MySQL deployments with the Percona Operator for MySQL. Get expert guidance for complex tasks like multi-cloud replication, database migration and building platforms.
|
Connect with our engineers and fellow users for general questions, troubleshooting, and sharing feedback and ideas.
|
+| **[Get Percona Support](https://hubs.ly/Q02ZTH940)** | **[Visit our Forum](https://forums.percona.com/c/mysql-mariadb/percona-kubernetes-operator-for-mysql/28)** |
+
# Contributing
Percona welcomes and encourages community contributions to help improve Percona Operator for MySQL.
See the [Contribution Guide](CONTRIBUTING.md) and [Building and Testing Guide](e2e-tests/README.md) for more information on how you can contribute.
-## Communication
-
-We would love to hear from you! Reach out to us on [Forum](https://forums.percona.com/c/mysql-mariadb/percona-kubernetes-operator-for-mysql/28) with your questions, feedback and ideas
-
-# Join Percona Kubernetes Squad!
-```
- % _____
- %%% | __ \
- ###%%%%%%%%%%%%* | |__) |__ _ __ ___ ___ _ __ __ _
- ### ##%% %%%% | ___/ _ \ '__/ __/ _ \| '_ \ / _` |
- #### ##% %%%% | | | __/ | | (_| (_) | | | | (_| |
- ### #### %%% |_| \___|_| \___\___/|_| |_|\__,_|
- ,((### ### %%% _ _ _____ _
- (((( (### #### %%%% | | / _ \ / ____| | |
- ((( ((# ###### | | _| (_) |___ | (___ __ _ _ _ __ _ __| |
- (((( (((# #### | |/ /> _
Date: Mon, 9 Dec 2024 20:47:34 +0200
Subject: [PATCH 36/44] resolving conflicts by accepting the target branch
---
README.md | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 186c8c43a..29d061e5a 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ While the Percona Operator is primarily managed through the command line, you ca
# Architecture
-Percona Operators are based on the [Operator SDK](https://github.com/operator-framework/operator-sdk) and leverage Kubernetes primitives to follow best CNCF practices.
+Percona Operators are based on the [Operator SDK](https://github.com/operator-framework/operator-sdk) and leverage Kubernetes primitives to follow best CNCF practices.
Please read more about [architecture and design decisions](https://www.percona.com/doc/kubernetes-operator-for-pxc/architecture.html).
@@ -35,7 +35,7 @@ To learn more about the Operator, check the [Percona Operator for MySQL based on
# Quickstart installation
-Ready to try out the Operator? Check the [Quickstart tutorial](https://docs.percona.com/percona-operator-for-mysql/pxc/quickstart.html) for easy-to follow steps.
+Ready to try out the Operator? Check the [Quickstart tutorial](https://docs.percona.com/percona-operator-for-mysql/pxc/quickstart.html) for easy-to follow steps.
Below is one of the ways to deploy the Operator using `kubectl`.
@@ -60,7 +60,7 @@ See full documentation with examples and various advanced cases on [percona.com]
**Commercial Support** | **Community Support** |
:-: | :-: |
-|
Enterprise-grade assistance for your mission-critical MySQL deployments with the Percona Operator for MySQL. Get expert guidance for complex tasks like multi-cloud replication, database migration and building platforms.
|
Connect with our engineers and fellow users for general questions, troubleshooting, and sharing feedback and ideas.
|
+|
Enterprise-grade assistance for your mission-critical MySQL deployments with the Percona Operator for MySQL. Get expert guidance for complex tasks like multi-cloud replication, database migration and building platforms.
|
Connect with our engineers and fellow users for general questions, troubleshooting, and sharing feedback and ideas.
|
| **[Get Percona Support](https://hubs.ly/Q02ZTH940)** | **[Visit our Forum](https://forums.percona.com/c/mysql-mariadb/percona-kubernetes-operator-for-mysql/28)** |
# Contributing
@@ -72,9 +72,10 @@ See the [Contribution Guide](CONTRIBUTING.md) and [Building and Testing Guide](e
## Roadmap
We have a public roadmap which can be found [here](https://github.com/orgs/percona/projects/10). Please feel free to contribute and propose new features by following the roadmap [guidelines](https://github.com/percona/roadmap).
-
+
## Submitting Bug Reports
-If you find a bug in Percona Docker Images or in one of the related projects, please submit a report to that project's [JIRA](https://jira.percona.com/browse/K8SPXC) issue tracker or [create a GitHub issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-an-issue#creating-an-issue-from-a-repository) in this repository.
+If you find a bug in Percona Docker Images or in one of the related projects, please submit a report to that project's [JIRA](https://jira.percona.com/browse/K8SPXC) issue tracker or [create a GitHub issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/creating-an-issue#creating-an-issue-from-a-repository) in this repository.
Learn more about submitting bugs, new features ideas and improvements in the [Contribution Guide](CONTRIBUTING.md).
+
From ffa45dd149388fb638ae9c3314093287e0411ae1 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Tue, 10 Dec 2024 10:35:06 +0200
Subject: [PATCH 37/44] refactor: update Jenkinsfile for improved readability
and consistency
---
Jenkinsfile | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 44aab95ab..52fb3df99 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1,4 +1,4 @@
-region='us-central1-a'
+region="us-central1-a"
testUrlPrefix="https://percona-jenkins-artifactory-public.s3.amazonaws.com/cloud-pxc-operator"
tests=[]
@@ -14,7 +14,7 @@ void createCluster(String CLUSTER_SUFFIX) {
gcloud config set project $GCP_PROJECT
gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
- gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 && \
+ gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias&& \
kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
if [ \${ret_val} -eq 0 ]; then break; fi
ret_num=\$((ret_num + 1))
@@ -150,7 +150,7 @@ void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) {
"""
}
-TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |'
+TestsReport = '| Test name | Status |\r\n| ------------- | ------------- |'
TestsReportXML = '\n'
void makeReport() {
@@ -158,9 +158,9 @@ void makeReport() {
def startedTestAmount = 0
for (int i=0; i
Date: Wed, 1 Jan 2025 15:56:51 +0200
Subject: [PATCH 38/44] Enhance Jenkins pipeline to support manual builds and
improve logging for previous builds
---
Jenkinsfile | 40 +++++++++++++++++++++++++++-------------
e2e-tests/build | 1 +
2 files changed, 28 insertions(+), 13 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 3e07456f2..7c08c12b5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -245,8 +245,18 @@ void runTest(Integer TEST_ID) {
}
}
+boolean isManualBuild() {
+ def causes = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')
+ return !causes.isEmpty()
+}
+
needToRunTests = true
void checkE2EIgnoreFiles() {
+ if (isManualBuild()) {
+ echo "This is a manual rebuild. Forcing pipeline execution."
+ return
+ }
+
def e2eignoreFile = ".e2eignore"
if (fileExists(e2eignoreFile)) {
def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
@@ -255,18 +265,14 @@ void checkE2EIgnoreFiles() {
def build = currentBuild.previousBuild
while (build != null) {
- if (build.result == 'SUCCESS') {
- try {
- echo "Found a previous successful build: $build.number"
- copyArtifacts(projectName: env.JOB_NAME, selector: specific("$build.number"), filter: "$lastProcessedCommitFile")
- lastProcessedCommitHash = readFile("$lastProcessedCommitFile").trim()
- echo "lastProcessedCommitHash: $lastProcessedCommitHash"
- break
- } catch (Exception e) {
- echo "No $lastProcessedCommitFile found in build $build.number. Checking earlier builds."
- }
- } else {
- echo "Build $build.number was not successful. Checking earlier builds."
+ try {
+ echo "Checking previous build: #$build.number"
+ copyArtifacts(projectName: env.JOB_NAME, selector: specific("$build.number"), filter: lastProcessedCommitFile)
+ lastProcessedCommitHash = readFile(lastProcessedCommitFile).trim()
+ echo "Last processed commit hash: $lastProcessedCommitHash"
+ break
+ } catch (Exception e) {
+ echo "No $lastProcessedCommitFile found in build $build.number. Checking earlier builds."
}
build = build.previousBuild
}
@@ -288,13 +294,21 @@ void checkE2EIgnoreFiles() {
if (needToRunTests) {
echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
} else {
- echo "All changed files are e2eignore files. Aborting pipeline execution."
+ if (currentBuild.previousBuild?.result in ['FAILURE', 'ABORTED', 'UNSTABLE']) {
+ echo "All changed files are e2eignore files, and previous build was unsuccessful. Propagating previous state."
+ currentBuild.result = currentBuild.previousBuild?.result
+ error "Skipping execution as non-significant changes detected and previous build was unsuccessful."
+ } else {
+ echo "All changed files are e2eignore files. Aborting pipeline execution."
+ }
}
sh """
echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
"""
archiveArtifacts "$lastProcessedCommitFile"
+ } else {
+ echo "No $e2eignoreFile file found. Proceeding with execution."
}
}
diff --git a/e2e-tests/build b/e2e-tests/build
index cb1ca1ef6..02fb878af 100755
--- a/e2e-tests/build
+++ b/e2e-tests/build
@@ -1,6 +1,7 @@
#!/bin/bash
set -o errexit
+set -o xtrace
test_dir="$(dirname $0)"
. $(dirname $0)/functions
From c99f533c967f1dbb8ad32870d17f6ef0a0fef282 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Tue, 7 Jan 2025 11:05:42 +0200
Subject: [PATCH 39/44] Extract node preparation steps into a function.
---
Jenkinsfile | 64 ++++++++++++++++++++++++++++-------------------------
1 file changed, 34 insertions(+), 30 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 7c08c12b5..105e0610c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -14,7 +14,7 @@ void createCluster(String CLUSTER_SUFFIX) {
gcloud config set project $GCP_PROJECT
gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
- gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias&& \
+ gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias && \
kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
if [ \${ret_val} -eq 0 ]; then break; fi
ret_num=\$((ret_num + 1))
@@ -145,7 +145,7 @@ void printKubernetesStatus(String LOCATION, String CLUSTER_SUFFIX) {
echo
kubectl top pod --all-namespaces
echo
- kubectl get events --field-selector type!=Normal --all-namespaces
+ kubectl get events --field-selector type!=Normal --all-namespaces --sort-by=".lastTimestamp"
echo "======================================================"
"""
}
@@ -228,6 +228,7 @@ void runTest(Integer TEST_ID) {
}
catch (exc) {
printKubernetesStatus("AFTER","$clusterSuffix")
+ echo "Test $testName has failed!"
if (retryCount >= 1 || currentBuild.nextBuild != null) {
currentBuild.result = 'FAILURE'
return true
@@ -245,6 +246,35 @@ void runTest(Integer TEST_ID) {
}
}
+void prepareNode() {
+ sh """
+ sudo curl -s -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && sudo chmod +x /usr/local/bin/kubectl
+ kubectl version --client --output=yaml
+
+ curl -fsSL https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz | sudo tar -C /usr/local/bin --strip-components 1 -xzf - linux-amd64/helm
+
+ sudo curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -o /usr/local/bin/yq && sudo chmod +x /usr/local/bin/yq
+ sudo curl -fsSL https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux64 -o /usr/local/bin/jq && sudo chmod +x /usr/local/bin/jq
+
+ sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOF
+[google-cloud-cli]
+name=Google Cloud CLI
+baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=0
+gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+EOF
+ sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
+
+ curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
+
+ sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true
+ sudo percona-release enable-only tools
+ sudo yum install -y percona-xtrabackup-80 | true
+ """
+}
+
boolean isManualBuild() {
def causes = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')
return !causes.isEmpty()
@@ -354,6 +384,7 @@ pipeline {
}
steps {
initTests()
+ prepareNode()
script {
if (AUTHOR_NAME == 'null') {
AUTHOR_NAME = sh(script: "git show -s --pretty=%ae | awk -F'@' '{print \$1}'", , returnStdout: true).trim()
@@ -366,33 +397,6 @@ pipeline {
}
}
}
- sh """
- sudo curl -s -L -o /usr/local/bin/kubectl https://dl.k8s.io/release/\$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && sudo chmod +x /usr/local/bin/kubectl
- kubectl version --client --output=yaml
-
- curl -fsSL https://get.helm.sh/helm-v3.12.3-linux-amd64.tar.gz | sudo tar -C /usr/local/bin --strip-components 1 -xzf - linux-amd64/helm
-
- sudo curl -fsSL https://github.com/mikefarah/yq/releases/download/v4.44.1/yq_linux_amd64 -o /usr/local/bin/yq && sudo chmod +x /usr/local/bin/yq
- sudo curl -fsSL https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux64 -o /usr/local/bin/jq && sudo chmod +x /usr/local/bin/jq
-
- sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOF
-[google-cloud-cli]
-name=Google Cloud CLI
-baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
-enabled=1
-gpgcheck=1
-repo_gpgcheck=0
-gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
-EOF
- sudo yum install -y google-cloud-cli google-cloud-cli-gke-gcloud-auth-plugin
-
- curl -sL https://github.com/mitchellh/golicense/releases/latest/download/golicense_0.2.0_linux_x86_64.tar.gz | sudo tar -C /usr/local/bin -xzf - golicense
-
- sudo yum install -y https://repo.percona.com/yum/percona-release-latest.noarch.rpm || true
- sudo percona-release enable-only tools
- sudo yum install -y percona-xtrabackup-80 | true
- """
-
withCredentials([file(credentialsId: 'cloud-secret-file', variable: 'CLOUD_SECRET_FILE')]) {
sh '''
cp $CLOUD_SECRET_FILE e2e-tests/conf/cloud-secret.yml
@@ -418,7 +422,7 @@ EOF
docker login -u '${USER}' -p '${PASS}'
export RELEASE=0
export IMAGE=\$DOCKER_TAG
- ./e2e-tests/build
+ e2e-tests/build
docker logout
"
sudo rm -rf ./build
From 643f307ddcc9865eabd290735518499d7055799a Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Fri, 10 Jan 2025 12:03:29 +0200
Subject: [PATCH 40/44] Update .e2eignore
---
.e2eignore | 2 ++
Jenkinsfile | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/.e2eignore b/.e2eignore
index 6366faa7f..b7442d7bb 100644
--- a/.e2eignore
+++ b/.e2eignore
@@ -1,3 +1,4 @@
+.github/**
docs/**
code-of-conduct.md
CONTRIBUTING.md
@@ -8,3 +9,4 @@ LICENSE
operator.png
kubernetes.svg
.e2eignore
+release_versions
\ No newline at end of file
diff --git a/Jenkinsfile b/Jenkinsfile
index 105e0610c..5b9fc4348 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -422,10 +422,10 @@ pipeline {
docker login -u '${USER}' -p '${PASS}'
export RELEASE=0
export IMAGE=\$DOCKER_TAG
- e2e-tests/build
+ ./e2e-tests/build
docker logout
"
- sudo rm -rf ./build
+ sudo rm -rf build
'''
}
stash includes: 'results/docker/TAG', name: 'IMAGE'
From 1b27819b0ad09f5016d3842d36cf5beb47db1423 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 11 Jan 2025 23:07:57 +0200
Subject: [PATCH 41/44] Refactor Jenkinsfile to improve e2eignore file handling
and streamline build process
---
Jenkinsfile | 89 +++++++++++++++++++++++++++--------------------------
1 file changed, 45 insertions(+), 44 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 5b9fc4348..b60118f5f 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -288,58 +288,59 @@ void checkE2EIgnoreFiles() {
}
def e2eignoreFile = ".e2eignore"
- if (fileExists(e2eignoreFile)) {
- def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
- def lastProcessedCommitFile="last-processed-commit.txt"
- def lastProcessedCommitHash = ""
-
- def build = currentBuild.previousBuild
- while (build != null) {
- try {
- echo "Checking previous build: #$build.number"
- copyArtifacts(projectName: env.JOB_NAME, selector: specific("$build.number"), filter: lastProcessedCommitFile)
- lastProcessedCommitHash = readFile(lastProcessedCommitFile).trim()
- echo "Last processed commit hash: $lastProcessedCommitHash"
- break
- } catch (Exception e) {
- echo "No $lastProcessedCommitFile found in build $build.number. Checking earlier builds."
- }
- build = build.previousBuild
- }
+ if ( ! fileExists(e2eignoreFile) ) {
+ echo "No $e2eignoreFile file found. Proceeding with execution."
+ return
+ }
- if (lastProcessedCommitHash == "") {
- echo "This is the first run. Using merge base as the starting point for the diff."
- changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n').findAll{it}
- } else {
- echo "Processing changes since last processed commit: $lastProcessedCommitHash"
- changedFiles = sh(script: "git diff --name-only $lastProcessedCommitHash HEAD", returnStdout: true).trim().split('\n').findAll{it}
+ def excludedFiles = readFile(e2eignoreFile).split('\n').collect{it.trim()}
+ def lastProcessedCommitFile = "last-processed-commit.txt"
+ def lastProcessedCommitHash = ""
+
+ def build = currentBuild.previousBuild
+ while (build != null) {
+ try {
+ echo "Checking previous build: #$build.number"
+ copyArtifacts(projectName: env.JOB_NAME, selector: specific("$build.number"), filter: lastProcessedCommitFile)
+ lastProcessedCommitHash = readFile(lastProcessedCommitFile).trim()
+ echo "Last processed commit hash: $lastProcessedCommitHash"
+ break
+ } catch (Exception e) {
+ echo "No $lastProcessedCommitFile found in build $build.number. Checking earlier builds."
}
+ build = build.previousBuild
+ }
+
+ if (lastProcessedCommitHash == "") {
+ echo "This is the first run. Using merge base as the starting point for the diff."
+ changedFiles = sh(script: "git diff --name-only \$(git merge-base HEAD origin/$CHANGE_TARGET)", returnStdout: true).trim().split('\n').findAll{it}
+ } else {
+ echo "Processing changes since last processed commit: $lastProcessedCommitHash"
+ changedFiles = sh(script: "git diff --name-only $lastProcessedCommitHash HEAD", returnStdout: true).trim().split('\n').findAll{it}
+ }
- echo "Excluded files: $excludedFiles"
- echo "Changed files: $changedFiles"
+ echo "Excluded files: $excludedFiles"
+ echo "Changed files: $changedFiles"
- def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
- needToRunTests = !changedFiles.every{changed -> excludedFilesRegex.any{regex -> changed ==~ regex}}
+ def excludedFilesRegex = excludedFiles.collect{it.replace("**", ".*").replace("*", "[^/]*")}
+ needToRunTests = !changedFiles.every{changed -> excludedFilesRegex.any{regex -> changed ==~ regex}}
- if (needToRunTests) {
- echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
+ if (needToRunTests) {
+ echo "Some changed files are outside of the e2eignore list. Proceeding with execution."
+ } else {
+ if (currentBuild.previousBuild?.result in ['FAILURE', 'ABORTED', 'UNSTABLE']) {
+ echo "All changed files are e2eignore files, and previous build was unsuccessful. Propagating previous state."
+ currentBuild.result = currentBuild.previousBuild?.result
+ error "Skipping execution as non-significant changes detected and previous build was unsuccessful."
} else {
- if (currentBuild.previousBuild?.result in ['FAILURE', 'ABORTED', 'UNSTABLE']) {
- echo "All changed files are e2eignore files, and previous build was unsuccessful. Propagating previous state."
- currentBuild.result = currentBuild.previousBuild?.result
- error "Skipping execution as non-significant changes detected and previous build was unsuccessful."
- } else {
- echo "All changed files are e2eignore files. Aborting pipeline execution."
- }
+ echo "All changed files are e2eignore files. Aborting pipeline execution."
}
-
- sh """
- echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
- """
- archiveArtifacts "$lastProcessedCommitFile"
- } else {
- echo "No $e2eignoreFile file found. Proceeding with execution."
}
+
+ sh """
+ echo \$(git rev-parse HEAD) > $lastProcessedCommitFile
+ """
+ archiveArtifacts "$lastProcessedCommitFile"
}
def isPRJob = false
From 940f31f41497550ddc945dd6d07333d549ebd68f Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 11 Jan 2025 23:15:01 +0200
Subject: [PATCH 42/44] Remove unnecessary blank line.
---
Jenkinsfile | 1 -
1 file changed, 1 deletion(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index b60118f5f..034849148 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -13,7 +13,6 @@ void createCluster(String CLUSTER_SUFFIX) {
gcloud auth activate-service-account --key-file $CLIENT_SECRET_FILE
gcloud config set project $GCP_PROJECT
gcloud container clusters list --filter $CLUSTER_NAME-${CLUSTER_SUFFIX} --zone $region --format='csv[no-heading](name)' | xargs gcloud container clusters delete --zone $region --quiet || true
-
gcloud container clusters create --zone $region $CLUSTER_NAME-${CLUSTER_SUFFIX} --cluster-version=1.28 --machine-type=n1-standard-4 --preemptible --disk-size 30 --num-nodes=\$NODES_NUM --network=jenkins-vpc --subnetwork=jenkins-${CLUSTER_SUFFIX} --no-enable-autoupgrade --cluster-ipv4-cidr=/21 --labels delete-cluster-after-hours=6 --enable-ip-alias && \
kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user jenkins@"$GCP_PROJECT".iam.gserviceaccount.com || ret_val=\$?
if [ \${ret_val} -eq 0 ]; then break; fi
From d0dba5e0d7b5063cd7a6345b2d88721ef0c18daa Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Sat, 11 Jan 2025 23:25:51 +0200
Subject: [PATCH 43/44] Fix paths in Jenkinsfile for docker tag file and
golicense command
---
Jenkinsfile | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 034849148..32bf1d21d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -415,7 +415,7 @@ pipeline {
withCredentials([usernamePassword(credentialsId: 'hub.docker.com', passwordVariable: 'PASS', usernameVariable: 'USER')]) {
sh '''
DOCKER_TAG=perconalab/percona-xtradb-cluster-operator:$VERSION
- docker_tag_file='./results/docker/TAG'
+ docker_tag_file='results/docker/TAG'
mkdir -p $(dirname ${docker_tag_file})
echo ${DOCKER_TAG} > "${docker_tag_file}"
sg docker -c "
@@ -482,7 +482,7 @@ pipeline {
withCredentials([string(credentialsId: 'GITHUB_API_TOKEN', variable: 'GITHUB_TOKEN')]) {
sh """
- golicense -plain ./percona-xtradb-cluster-operator \
+ golicense -plain percona-xtradb-cluster-operator \
| grep -v 'license not found' \
| sed -r 's/^[^ ]+[ ]+//' \
| sort \
From 6c055b73d96d0f0f5e8fda1289276dfb9fb1b019 Mon Sep 17 00:00:00 2001
From: Pavel Tankov <4014969+ptankov@users.noreply.github.com>
Date: Tue, 14 Jan 2025 18:21:56 +0200
Subject: [PATCH 44/44] remove unnecessary debug flag
---
e2e-tests/build | 1 -
1 file changed, 1 deletion(-)
diff --git a/e2e-tests/build b/e2e-tests/build
index 02fb878af..cb1ca1ef6 100755
--- a/e2e-tests/build
+++ b/e2e-tests/build
@@ -1,7 +1,6 @@
#!/bin/bash
set -o errexit
-set -o xtrace
test_dir="$(dirname $0)"
. $(dirname $0)/functions