diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index cb3f69a0..aaa9ae4b 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -16,28 +16,39 @@ import groovy.cli.commons.* * * usage: PackageBuildOutputs.groovy [options] * - * -w,--workDir Absolute path to the DBB build - * output directory + * -w,--workDir Absolute path to the DBB build + * output directory + * -properties,--packagingPropertiesFile Absolute path of a property file + * containing application specific + * packaging details. + * * Optional: - * -t,--tarFileName Name of the package tar file. - * (Optional) - * -d,--deployTypes Comma-seperated list of deployTypes - * to filter on the scope of the tar - * file. (Optional) - * -verb,--verbose Flag to provide more log output. - * (Optional) + * -t,--tarFileName Name of the package tar file. + * (Optional) + * -d,--deployTypes Comma-seperated list of deployTypes + * to filter on the scope of the tar + * file. (Optional) + * -verb,--verbose Flag to provide more log output. + * (Optional) + * -il,--includeLogs Comma-separated list of files/patterns + * from the USS build workspace * * Optional Artifactory Upload opts: * - * -p,--publish Flag to indicate package upload to - * the provided Artifactory server. - * (Optional) - * -prop,--propertyFile Absolute path of a property file - * containing application specific - * Artifactory details. (Optional) - * -v,--versionName Name of the Artifactory version. - * (Optional) - * -h,--help Prints this message + * -p,--publish Flag to indicate package upload to + * the provided Artifactory server. + * (Optional) + * -artifactory, + * --artifactoryPropertiesFile Absolute path of a property file + * containing application specific + * Artifactory details. (Optional) + * -v,--versionName Name of the Artifactory version. + * (Optional) + * -prop,--propertyFile ** Deprecated ** Absolute path of a + * property file containing application + * specific Artifactory details. (Optional) + * + * -h,--help Prints this message * * Version 0 - 2019 * called PublishLoadModule.groovy and located in Build/PublishLoadModules @@ -45,28 +56,34 @@ import groovy.cli.commons.* * Version 1 - 2021 * Re-Design to run as a post-build script and make publishing optional * + * Version 2 - 2022-02 + * - Externalize the Map of LLQ to CopyMode + * - Add capablity to add additional files from build workspace + * - Verbose logging will print tar contents + * ************************************************************************************/ // start create & publish package @Field Properties props = null props = parseInput(args) -// Map of last level dataset qualifier to DBB CopyToFS CopyMode. -// TODO: Customize to your needs. -def copyModeMap = ["COPYBOOK": CopyMode.TEXT, "COPY": CopyMode.TEXT, "DBRM": CopyMode.BINARY, "LOAD": CopyMode.LOAD] - - def startTime = new Date() props.startTime = startTime.format("yyyyMMdd.hhmmss.mmm") println("** PackageBuildOutputs start at $props.startTime") println("** Properties at startup:") props.each{k,v-> - println " $k -> $v" + if ( k == "artifactory.password" ) + println " $k -> xxxxxx " + else + println " $k -> $v" } // Enable file tagging BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging +// Map of last level dataset qualifier to DBB CopyToFS CopyMode. +def copyModeMap = evaluate(props.copyModeMap) + // read build report data println("** Read build report data from $props.workDir/BuildReport.json") def jsonOutputFile = new File("${props.workDir}/BuildReport.json") @@ -123,8 +140,6 @@ else { def String tarFileLabel = buildInfo[0].label def String tarFileName = (props.tarFileName) ? props.tarFileName : "${buildInfo[0].label}.tar" - def String buildGroup = buildInfo[0].group - //Create a temporary directory on zFS to copy the load modules from data sets to def tempLoadDir = new File("$props.workDir/tempPackageDir") @@ -160,7 +175,7 @@ else { // set copyMode based on last level qualifier currentCopyMode = copyModeMap[dataset.replaceAll(/.*\.([^.]*)/, "\$1")] - copy.setCopyMode(currentCopyMode) + copy.setCopyMode(DBBConstants.CopyMode.valueOf(currentCopyMode)) copy.setDataset(dataset) println " Copying $dataset($member) to $filePath with DBB Copymode $currentCopyMode" @@ -203,12 +218,39 @@ else { "-c", "tar rUXf $tarFile BuildReport.json" ] - + rc = runProcess(processCmd, new File(props.workDir)) assert rc == 0 : "Failed to append BuildReport.json" + //Package additional outputs to tar file. + if (props.includeLogs) (props.includeLogs).split(",").each { logPattern -> + println("** Adding $logPattern to $tarFile.") + processCmd = [ + "sh", + "-c", + "tar rUXf $tarFile $logPattern" + ] + + rc = runProcess(processCmd, new File(props.workDir)) + assert rc == 0 : "Failed to append $logPattern" + } + println ("** Package successfully created at $tarFile.") - + + if(props.verbose && props.verbose.toBoolean()) { + println ("** List package contents.") + + processCmd = [ + "sh", + "-c", + "tar tvf $tarFile" + ] + + rc = runProcess(processCmd, new File(props.workDir)) + assert rc == 0 : "Failed to list contents of tarfile $tarFile." + + } + //Set up the artifactory information to publish the tar file if (props.publish && props.publish.toBoolean()){ // Configuring ArtifactoryHelper parms @@ -272,14 +314,22 @@ def runProcess(ArrayList cmd, File dir){ */ def parseInput(String[] cliArgs){ def cli = new CliBuilder(usage: "PackageBuildOutputs.groovy [options]") + // required packaging options cli.w(longOpt:'workDir', args:1, argName:'dir', 'Absolute path to the DBB build output directory') + cli.properties(longOpt:'packagingPropertiesFile', args:1, argName:'packagingPropertiesFile', 'Path of a property file containing application specific packaging details.') + + // optional packaging options cli.d(longOpt:'deployTypes', args:1, argName:'deployTypes','Comma-seperated list of deployTypes to filter on the scope of the tar file. (Optional)') cli.t(longOpt:'tarFileName', args:1, argName:'filename', 'Name of the package tar file. (Optional)') + cli.il(longOpt:'includeLogs', args:1, argName:'includeLogs', 'Comma-separated list of files/patterns from the USS build workspace. (Optional)') // Artifactory Options: cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifactory server. (Optional)') cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the Artifactory version. (Optional)') - cli.prop(longOpt:'propertyFile', args:1, argName:'propertyFile', 'Absolute path of a property file containing application specific Artifactory details. (Optional)') + cli.artifactory(longOpt:'artifactoryPropertiesFile', args:1, argName:'artifactoryPropertiesFile', 'Path of a property file containing application specific Artifactory details. (Optional)') + // old prop option (deprecated) + cli.prop(longOpt:'propertyFile', args:1, argName:'propertyFile', 'Path of a property file containing application specific Artifactory details. (Optional) ** (Deprecated)') + cli.verb(longOpt:'verbose', 'Flag to provide more log output. (Optional)') cli.h(longOpt:'help', 'Prints this message') @@ -291,10 +341,25 @@ def parseInput(String[] cliArgs){ def props = new Properties() + // read properties file + if (opts.properties){ + def propertiesFile = new File(opts.properties) + if (propertiesFile.exists()){ + propertiesFile.withInputStream { props.load(it) } + } + } else { // read default sample properties file shipped with the script + def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent + def defaultPackagePropFile = new File("$scriptDir/packageBuildOutputs.properties") + if (defaultPackagePropFile.exists()){ + defaultPackagePropFile.withInputStream { props.load(it) } + } + } + // set command line arguments if (opts.w) props.workDir = opts.w if (opts.d) props.deployTypeFilter = opts.d if (opts.t) props.tarFileName = opts.t + if (opts.il) props.includeLogs = opts.il props.verbose = (opts.verb) ? 'true' : 'false' @@ -302,6 +367,15 @@ def parseInput(String[] cliArgs){ if (opts.v) props.versionName = opts.v props.publish = (opts.p) ? 'true' : 'false' + // Optional artifactory properties + if (opts.artifactory){ + def propertyFile = new File(opts.artifactory) + if (propertyFile.exists()){ + propertyFile.withInputStream { props.load(it) } + } + } + + // ** Deprecated ** Read of artifactory properties if (opts.prop){ def propertyFile = new File(opts.prop) if (propertyFile.exists()){ @@ -312,6 +386,7 @@ def parseInput(String[] cliArgs){ // validate required props try { assert props.workDir : "Missing property build work directory" + assert props.copyModeMap : "Missing property package.copyModeMap" if (props.publish && props.publish.toBoolean()){ assert props.get("artifactory.url") : "Missing Artifactory URL" assert props.get("artifactory.repo") : "Missing Artifactory Repository" diff --git a/Pipeline/PackageBuildOutputs/README.md b/Pipeline/PackageBuildOutputs/README.md index 055b6b1f..cd963a97 100644 --- a/Pipeline/PackageBuildOutputs/README.md +++ b/Pipeline/PackageBuildOutputs/README.md @@ -16,7 +16,7 @@ The ArtifactoryHelpers is a very simple implementation sufficient for a show cas 1. After a successful DBB build, `PackageBuildOutputs.groovy` reads the build report and retrieves all outputs from the build report. It excludes outputs without a `deployType` as well as those labeled `ZUNIT-TESTCASE` 2. It then invokes CopyToHFS API to copy the outputs from the libraries to a temporary directory on zFS. It will set the file tags based on the ZLANG setting (Note: A workaround is implemented to tag files as binary); all files require to be tagged. Please check the COPYMODE list, which maps last level qualifiers to the copymode of CopyToHFS. -3. It packages these load files into a tar file, and adds the BuildReport.json to it. +3. It packages these load files into a tar file, and adds the BuildReport.json and optionally other build logs from the build workspace. 4. (Optional) Publishes the tar file to the Artifactory repository based on the given configuration using the ArtifactoryHelpers. ## Invocation samples @@ -50,9 +50,45 @@ groovyz /var/jenkins/pipeline/PackageBuildOutputs.groovy --workDir /var/jenkins/ ** Build finished ``` +### Package only including *.log files from build workspace + +``` +groovyz dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy --workDir /var/jenkins/workspace/MortgageApplication/BUILD-2 --packagingPropertiesFile dbb/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties --includeLogs *.log + +** PackageBuildOutputs start at 20220222.112956.029 +** Properties at startup: + verbose -> false + copyModeMap -> ["COPYBOOK": "TEXT", "COPY": "TEXT", "DBRM": "BINARY", "LOAD": "LOAD"] + startTime -> 20220222.112956.029 + publish -> false + includeLogs -> *.log + workDir -> /var/jenkins/workspace/MortgageApplication/BUILD-2 +** Read build report data from /var/jenkins/workspace/MortgageApplication/BUILD-2/BuildReport.json +** Removing Output Records without deployType or with deployType=ZUNIT-TESTCASE +** Copying BuildOutputs to temporary package dir. +*** Number of build outputs to publish: 8 + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMORT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMLIS) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSCSMRT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMPMT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSCMORT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSCSMRD) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMLIST) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD + Copying JENKINS.DBB.SAMP.BUILD.DBRM(EPSCMORT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.DBRM with DBB Copymode BINARY +** Creating tar file at /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar. + +** Adding BuildReport.json to /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar. + +** Adding *.log to /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar. + +** Package successfully created at /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar. +** Build finished +``` + + ### Package and Publish to Artifactory ``` -groovyz /var/jenkins/pipeline/PublishLoadModule.groovy --workDir /var/jenkins/workspace/MortgageApplication/build.20210727.073406.034 --propertyFile publish.properties -v MortgageRelease_1.0 -t myPackage.tar --verbose --publish +groovyz /var/jenkins/pipeline/PublishLoadModule.groovy --workDir /var/jenkins/workspace/MortgageApplication/build.20210727.073406.034 --artifactoryPropertiesFile publish.properties -v MortgageRelease_1.0 -t myPackage.tar --verbose --publish ** PackageBuildOutputs start at 20210727.042032.020 @@ -108,32 +144,39 @@ groovyz /var/jenkins/pipeline/ArtifactoryHelpers.groovy --url http://10.3.20.23 ## Command Line Options Summary - PackageBuildOutputs ``` -usage: PackageBuildOutputs.groovy [options] - - -w,--workDir Absolute path to the DBB build - output directory - -Optional: - -t,--tarFileName Name of the package tar file. - (Optional) - -d,--deployTypes Comma-seperated list of deployTypes - to filter on the scope of the tar - file. (Optional) - -verb,--verbose Flag to provide more log output. - (Optional) - -Artifactory Upload Options - - -p,--publish Flag to indicate package upload to - the provided Artifactory server. - (Optional) - -prop,--propertyFile Absolute path of a property file - containing application specific - Artifactory details. (Optional) - -v,--versionName Name of the Artifactory version. - (Optional) - - -h,--help Prints this message + usage: PackageBuildOutputs.groovy [options] + + -w,--workDir Absolute path to the DBB build + output directory + -properties,--packagingPropertiesFile Absolute path of a property file + containing application specific + packaging details. + + Optional: + -t,--tarFileName Name of the package tar file. + (Optional) + -d,--deployTypes Comma-seperated list of deployTypes + to filter on the scope of the tar + file. (Optional) + -verb,--verbose Flag to provide more log output. + (Optional) + -il,--includeLogs Comma-separated list of files/patterns + from the USS build workspace + + Optional Artifactory Upload opts: + + -p,--publish Flag to indicate package upload to + the provided Artifactory server. + (Optional) + -artifactory, + --artifactoryPropertiesFile Absolute path of a property file + containing application specific + Artifactory details. (Optional) + -v,--versionName Name of the Artifactory version. + (Optional) + + + -h,--help Prints this message ``` ## Command Line Options Summary - ArtifactoryHelpers diff --git a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties new file mode 100644 index 00000000..de47d9d1 --- /dev/null +++ b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties @@ -0,0 +1,21 @@ +######################################################################## +### The following properties are configure the packaging step +######################################################################## + +# required mapping of last level dataset qualifier to DBB CopyToFS CopyMode +# DBB supported copy modes are documented at +# https://www.ibm.com/docs/api/v1/content/SS6T76_1.1.0/javadoc/com/ibm/dbb/build/DBBConstants.CopyMode.html +copyModeMap = ["COPYBOOK": "TEXT", "COPY": "TEXT", "DBRM": "BINARY", "LOAD": "LOAD"] + +# Comma-separated list of deployTypes to limit the scope of the tar +# file to a subset of the build outputs. (Optional) +# Please not that the cli option `deployTypes` overwrites this setting +# Sample: deployTypesFilter=CICSLOAD,BATCHLOAD +# +# deployTypesFilter= + +# Comma-separated list of files/patterns from the USS build workspace. (Optional) +# Please not that the cli option `includeLogs` overwrites this setting +# Sample: includeLogs = *.log,*.xml +# +# includeLogs = \ No newline at end of file