Skip to content

Commit

Permalink
Enhance PackageBuildOutputs sample (#100)
Browse files Browse the repository at this point in the history 
* externalize copymodes and allow packaging of additional build logs

* Update readme

* implements #97 to mask artifactory password

* Cli option -prop marked as deprecated
  • Loading branch information
@dennis-behm
dennis-behm authored Feb 28, 2022
1 parent d60fc8c commit 1f3aa16
Show file tree
Hide file tree
Showing 3 changed files with 197 additions and 58 deletions.
135 changes: 105 additions & 30 deletions Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,57 +16,74 @@ import groovy.cli.commons.*
*
* usage: PackageBuildOutputs.groovy [options]
*
* -w,--workDir <dir> Absolute path to the DBB build
* output directory
* -w,--workDir <dir> Absolute path to the DBB build
* output directory
* -properties,--packagingPropertiesFile <file> Absolute path of a property file
* containing application specific
* packaging details.
*
* Optional:
* -t,--tarFileName <filename> Name of the package tar file.
* (Optional)
* -d,--deployTypes <deployTypes> Comma-seperated list of deployTypes
* to filter on the scope of the tar
* file. (Optional)
* -verb,--verbose Flag to provide more log output.
* (Optional)
* -t,--tarFileName <filename> Name of the package tar file.
* (Optional)
* -d,--deployTypes <deployTypes> Comma-seperated list of deployTypes
* to filter on the scope of the tar
* file. (Optional)
* -verb,--verbose Flag to provide more log output.
* (Optional)
* -il,--includeLogs Comma-separated list of files/patterns
* from the USS build workspace
*
* Optional Artifactory Upload opts:
*
* -p,--publish Flag to indicate package upload to
* the provided Artifactory server.
* (Optional)
* -prop,--propertyFile <propertyFile> Absolute path of a property file
* containing application specific
* Artifactory details. (Optional)
* -v,--versionName <versionName> Name of the Artifactory version.
* (Optional)
* -h,--help Prints this message
* -p,--publish Flag to indicate package upload to
* the provided Artifactory server.
* (Optional)
* -artifactory,
* --artifactoryPropertiesFile <propertyFile> Absolute path of a property file
* containing application specific
* Artifactory details. (Optional)
* -v,--versionName <versionName> Name of the Artifactory version.
* (Optional)
* -prop,--propertyFile <propertyFile> ** Deprecated ** Absolute path of a
* property file containing application
* specific Artifactory details. (Optional)
*
* -h,--help Prints this message
*
* Version 0 - 2019
* called PublishLoadModule.groovy and located in Build/PublishLoadModules
*
* Version 1 - 2021
* Re-Design to run as a post-build script and make publishing optional
*
* Version 2 - 2022-02
* - Externalize the Map of LLQ to CopyMode
* - Add capablity to add additional files from build workspace
* - Verbose logging will print tar contents
*
************************************************************************************/

// start create & publish package
@Field Properties props = null
props = parseInput(args)

// Map of last level dataset qualifier to DBB CopyToFS CopyMode.
// TODO: Customize to your needs.
def copyModeMap = ["COPYBOOK": CopyMode.TEXT, "COPY": CopyMode.TEXT, "DBRM": CopyMode.BINARY, "LOAD": CopyMode.LOAD]


def startTime = new Date()
props.startTime = startTime.format("yyyyMMdd.hhmmss.mmm")
println("** PackageBuildOutputs start at $props.startTime")
println("** Properties at startup:")
props.each{k,v->
println " $k -> $v"
if ( k == "artifactory.password" )
println " $k -> xxxxxx "
else
println " $k -> $v"
}

// Enable file tagging
BuildProperties.setProperty("dbb.file.tagging", "true") // Enable dbb file tagging

// Map of last level dataset qualifier to DBB CopyToFS CopyMode.
def copyModeMap = evaluate(props.copyModeMap)

// read build report data
println("** Read build report data from $props.workDir/BuildReport.json")
def jsonOutputFile = new File("${props.workDir}/BuildReport.json")
Expand Down Expand Up @@ -123,8 +140,6 @@ else {

def String tarFileLabel = buildInfo[0].label
def String tarFileName = (props.tarFileName) ? props.tarFileName : "${buildInfo[0].label}.tar"
def String buildGroup = buildInfo[0].group


//Create a temporary directory on zFS to copy the load modules from data sets to
def tempLoadDir = new File("$props.workDir/tempPackageDir")
Expand Down Expand Up @@ -160,7 +175,7 @@ else {

// set copyMode based on last level qualifier
currentCopyMode = copyModeMap[dataset.replaceAll(/.*\.([^.]*)/, "\$1")]
copy.setCopyMode(currentCopyMode)
copy.setCopyMode(DBBConstants.CopyMode.valueOf(currentCopyMode))
copy.setDataset(dataset)

println " Copying $dataset($member) to $filePath with DBB Copymode $currentCopyMode"
Expand Down Expand Up @@ -203,12 +218,39 @@ else {
"-c",
"tar rUXf $tarFile BuildReport.json"
]

rc = runProcess(processCmd, new File(props.workDir))
assert rc == 0 : "Failed to append BuildReport.json"

//Package additional outputs to tar file.
if (props.includeLogs) (props.includeLogs).split(",").each { logPattern ->
println("** Adding $logPattern to $tarFile.")
processCmd = [
"sh",
"-c",
"tar rUXf $tarFile $logPattern"
]

rc = runProcess(processCmd, new File(props.workDir))
assert rc == 0 : "Failed to append $logPattern"
}

println ("** Package successfully created at $tarFile.")


if(props.verbose && props.verbose.toBoolean()) {
println ("** List package contents.")

processCmd = [
"sh",
"-c",
"tar tvf $tarFile"
]

rc = runProcess(processCmd, new File(props.workDir))
assert rc == 0 : "Failed to list contents of tarfile $tarFile."

}

//Set up the artifactory information to publish the tar file
if (props.publish && props.publish.toBoolean()){
// Configuring ArtifactoryHelper parms
Expand Down Expand Up @@ -272,14 +314,22 @@ def runProcess(ArrayList cmd, File dir){
*/
def parseInput(String[] cliArgs){
def cli = new CliBuilder(usage: "PackageBuildOutputs.groovy [options]")
// required packaging options
cli.w(longOpt:'workDir', args:1, argName:'dir', 'Absolute path to the DBB build output directory')
cli.properties(longOpt:'packagingPropertiesFile', args:1, argName:'packagingPropertiesFile', 'Path of a property file containing application specific packaging details.')

// optional packaging options
cli.d(longOpt:'deployTypes', args:1, argName:'deployTypes','Comma-seperated list of deployTypes to filter on the scope of the tar file. (Optional)')
cli.t(longOpt:'tarFileName', args:1, argName:'filename', 'Name of the package tar file. (Optional)')
cli.il(longOpt:'includeLogs', args:1, argName:'includeLogs', 'Comma-separated list of files/patterns from the USS build workspace. (Optional)')

// Artifactory Options:
cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifactory server. (Optional)')
cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the Artifactory version. (Optional)')
cli.prop(longOpt:'propertyFile', args:1, argName:'propertyFile', 'Absolute path of a property file containing application specific Artifactory details. (Optional)')
cli.artifactory(longOpt:'artifactoryPropertiesFile', args:1, argName:'artifactoryPropertiesFile', 'Path of a property file containing application specific Artifactory details. (Optional)')
// old prop option (deprecated)
cli.prop(longOpt:'propertyFile', args:1, argName:'propertyFile', 'Path of a property file containing application specific Artifactory details. (Optional) ** (Deprecated)')

cli.verb(longOpt:'verbose', 'Flag to provide more log output. (Optional)')

cli.h(longOpt:'help', 'Prints this message')
Expand All @@ -291,17 +341,41 @@ def parseInput(String[] cliArgs){

def props = new Properties()

// read properties file
if (opts.properties){
def propertiesFile = new File(opts.properties)
if (propertiesFile.exists()){
propertiesFile.withInputStream { props.load(it) }
}
} else { // read default sample properties file shipped with the script
def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent
def defaultPackagePropFile = new File("$scriptDir/packageBuildOutputs.properties")
if (defaultPackagePropFile.exists()){
defaultPackagePropFile.withInputStream { props.load(it) }
}
}

// set command line arguments
if (opts.w) props.workDir = opts.w
if (opts.d) props.deployTypeFilter = opts.d
if (opts.t) props.tarFileName = opts.t
if (opts.il) props.includeLogs = opts.il

props.verbose = (opts.verb) ? 'true' : 'false'

// Optional Artifactory to publish
if (opts.v) props.versionName = opts.v
props.publish = (opts.p) ? 'true' : 'false'

// Optional artifactory properties
if (opts.artifactory){
def propertyFile = new File(opts.artifactory)
if (propertyFile.exists()){
propertyFile.withInputStream { props.load(it) }
}
}

// ** Deprecated ** Read of artifactory properties
if (opts.prop){
def propertyFile = new File(opts.prop)
if (propertyFile.exists()){
Expand All @@ -312,6 +386,7 @@ def parseInput(String[] cliArgs){
// validate required props
try {
assert props.workDir : "Missing property build work directory"
assert props.copyModeMap : "Missing property package.copyModeMap"
if (props.publish && props.publish.toBoolean()){
assert props.get("artifactory.url") : "Missing Artifactory URL"
assert props.get("artifactory.repo") : "Missing Artifactory Repository"
Expand Down
99 changes: 71 additions & 28 deletions Pipeline/PackageBuildOutputs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ The ArtifactoryHelpers is a very simple implementation sufficient for a show cas

1. After a successful DBB build, `PackageBuildOutputs.groovy` reads the build report and retrieves all outputs from the build report. It excludes outputs without a `deployType` as well as those labeled `ZUNIT-TESTCASE`
2. It then invokes CopyToHFS API to copy the outputs from the libraries to a temporary directory on zFS. It will set the file tags based on the ZLANG setting (Note: A workaround is implemented to tag files as binary); all files require to be tagged. Please check the COPYMODE list, which maps last level qualifiers to the copymode of CopyToHFS.
3. It packages these load files into a tar file, and adds the BuildReport.json to it.
3. It packages these load files into a tar file, and adds the BuildReport.json and optionally other build logs from the build workspace.
4. (Optional) Publishes the tar file to the Artifactory repository based on the given configuration using the ArtifactoryHelpers.

## Invocation samples
Expand Down Expand Up @@ -50,9 +50,45 @@ groovyz /var/jenkins/pipeline/PackageBuildOutputs.groovy --workDir /var/jenkins/
** Build finished
```

### Package only including *.log files from build workspace

```
groovyz dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy --workDir /var/jenkins/workspace/MortgageApplication/BUILD-2 --packagingPropertiesFile dbb/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties --includeLogs *.log
** PackageBuildOutputs start at 20220222.112956.029
** Properties at startup:
verbose -> false
copyModeMap -> ["COPYBOOK": "TEXT", "COPY": "TEXT", "DBRM": "BINARY", "LOAD": "LOAD"]
startTime -> 20220222.112956.029
publish -> false
includeLogs -> *.log
workDir -> /var/jenkins/workspace/MortgageApplication/BUILD-2
** Read build report data from /var/jenkins/workspace/MortgageApplication/BUILD-2/BuildReport.json
** Removing Output Records without deployType or with deployType=ZUNIT-TESTCASE
** Copying BuildOutputs to temporary package dir.
*** Number of build outputs to publish: 8
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMORT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMLIS) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSCSMRT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMPMT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSCMORT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSCSMRD) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.LOAD(EPSMLIST) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.LOAD with DBB Copymode LOAD
Copying JENKINS.DBB.SAMP.BUILD.DBRM(EPSCMORT) to /var/jenkins/workspace/MortgageApplication/BUILD-2/tempPackageDir/JENKINS.DBB.SAMP.BUILD.DBRM with DBB Copymode BINARY
** Creating tar file at /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar.
** Adding BuildReport.json to /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar.
** Adding *.log to /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar.
** Package successfully created at /var/jenkins/workspace/MortgageApplication/BUILD-2/build.20220222.021034.010.tar.
** Build finished
```


### Package and Publish to Artifactory
```
groovyz /var/jenkins/pipeline/PublishLoadModule.groovy --workDir /var/jenkins/workspace/MortgageApplication/build.20210727.073406.034 --propertyFile publish.properties -v MortgageRelease_1.0 -t myPackage.tar --verbose --publish
groovyz /var/jenkins/pipeline/PublishLoadModule.groovy --workDir /var/jenkins/workspace/MortgageApplication/build.20210727.073406.034 --artifactoryPropertiesFile publish.properties -v MortgageRelease_1.0 -t myPackage.tar --verbose --publish
** PackageBuildOutputs start at 20210727.042032.020
Expand Down Expand Up @@ -108,32 +144,39 @@ groovyz /var/jenkins/pipeline/ArtifactoryHelpers.groovy --url http://10.3.20.23
## Command Line Options Summary - PackageBuildOutputs

```
usage: PackageBuildOutputs.groovy [options]
-w,--workDir <dir> Absolute path to the DBB build
output directory
Optional:
-t,--tarFileName <filename> Name of the package tar file.
(Optional)
-d,--deployTypes <deployTypes> Comma-seperated list of deployTypes
to filter on the scope of the tar
file. (Optional)
-verb,--verbose Flag to provide more log output.
(Optional)
Artifactory Upload Options
-p,--publish Flag to indicate package upload to
the provided Artifactory server.
(Optional)
-prop,--propertyFile <propertyFile> Absolute path of a property file
containing application specific
Artifactory details. (Optional)
-v,--versionName <versionName> Name of the Artifactory version.
(Optional)
-h,--help Prints this message
usage: PackageBuildOutputs.groovy [options]
-w,--workDir <dir> Absolute path to the DBB build
output directory
-properties,--packagingPropertiesFile <file> Absolute path of a property file
containing application specific
packaging details.
Optional:
-t,--tarFileName <filename> Name of the package tar file.
(Optional)
-d,--deployTypes <deployTypes> Comma-seperated list of deployTypes
to filter on the scope of the tar
file. (Optional)
-verb,--verbose Flag to provide more log output.
(Optional)
-il,--includeLogs Comma-separated list of files/patterns
from the USS build workspace
Optional Artifactory Upload opts:
-p,--publish Flag to indicate package upload to
the provided Artifactory server.
(Optional)
-artifactory,
--artifactoryPropertiesFile <propertyFile> Absolute path of a property file
containing application specific
Artifactory details. (Optional)
-v,--versionName <versionName> Name of the Artifactory version.
(Optional)
-h,--help Prints this message
```

## Command Line Options Summary - ArtifactoryHelpers
Expand Down
21 changes: 21 additions & 0 deletions Pipeline/PackageBuildOutputs/packageBuildOutputs.properties
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
########################################################################
### The following properties are configure the packaging step
########################################################################

# required mapping of last level dataset qualifier to DBB CopyToFS CopyMode
# DBB supported copy modes are documented at
# https://www.ibm.com/docs/api/v1/content/SS6T76_1.1.0/javadoc/com/ibm/dbb/build/DBBConstants.CopyMode.html
copyModeMap = ["COPYBOOK": "TEXT", "COPY": "TEXT", "DBRM": "BINARY", "LOAD": "LOAD"]

# Comma-separated list of deployTypes to limit the scope of the tar
# file to a subset of the build outputs. (Optional)
# Please not that the cli option `deployTypes` overwrites this setting
# Sample: deployTypesFilter=CICSLOAD,BATCHLOAD
#
# deployTypesFilter=

# Comma-separated list of files/patterns from the USS build workspace. (Optional)
# Please not that the cli option `includeLogs` overwrites this setting
# Sample: includeLogs = *.log,*.xml
#
# includeLogs =

0 comments on commit 1f3aa16

Please sign in to comment.