From 901593964d6f320c184df9d69501e70f518085b9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 8 Jun 2020 10:34:27 +0200 Subject: [PATCH 1/4] pipeline-integration-scripts --- Pipeline-Integration/README.md | 2 + .../UCD-Packaging/dbb-ucd-packaging.groovy | 263 ++++++++++++++++++ .../UCD-Packaging/dbb-ucd-packaging.md | 44 +++ 3 files changed, 309 insertions(+) create mode 100644 Pipeline-Integration/README.md create mode 100644 Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy create mode 100644 Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.md diff --git a/Pipeline-Integration/README.md b/Pipeline-Integration/README.md new file mode 100644 index 00000000..f34c88ee --- /dev/null +++ b/Pipeline-Integration/README.md @@ -0,0 +1,2 @@ +# Pipeline Integration Samples +This category contains examples of using DBB APIs to integrate additional parts of the pipeline. \ No newline at end of file diff --git a/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy b/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy new file mode 100644 index 00000000..cf00b1a1 --- /dev/null +++ b/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy @@ -0,0 +1,263 @@ +import com.ibm.dbb.build.* +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.report.* +import com.ibm.dbb.build.report.records.* +import groovy.time.* +import groovy.xml.MarkupBuilder +/** + * This script creates a version in UrbanCode Deploy based on the build result. + * + * usage: ucd-packaging.groovy [options] + * + * options: + * -b,--buztool Absolute path to UrbanCode Deploy buztool.sh script + * -w,--workDir Absolute path to the DBB build output directory + * -c,--component Name of the UCD component to create version in + * -v,--versionName Name of the UCD component version name (Optional) + * -h,--help Prints this message + * -ar,--artifactRepository Absolute path to Artifact Respository Server connection file (Optional) + * -p,--preview Preview, not executing buztool.sh + * + * notes: + * This script uses ship list specification and buztool parameters which are + * introduced since UCD v7.0.4. + * + * When used with an earlier version of UCD, please + * modify the script to remove the code that creates the top level property and + * the code that uses the -o buztool parameter. + * + * Version 0 - 2019 + * originally called deploy.groovy + * + * Version 1 - 2020-01 + * New option to support new UCD + Artifactory Support + * + * Version 2 - 2020-06 + * Added option to define UCD component version name (optional) + * Option -ar now optional; renamed to --artifactRepository, now supporting both Artifactory + UCD Codestation + * Added preview option to skip buztool.sh execution + * + */ + +// start create version +def properties = parseInput(args) +def startTime = new Date() +properties.startTime = startTime.format("yyyyMMdd.hhmmss.mmm") +println("** Create version start at $properties.startTime") +println("** Properties at startup:") +properties.each{k,v-> + println " $k -> $v" +} + +// read build report data +println("** Read build report data from $properties.workDir/BuildReport.json") +def jsonOutputFile = new File("${properties.workDir}/BuildReport.json") + +if(!jsonOutputFile.exists()){ + println("** Build report data at $properties.workDir/BuildReport.json not found") + System.exit() +} + +def buildReport= BuildReport.parse(new FileInputStream(jsonOutputFile)) + +// parse build report to find the build result meta info +def buildResult = buildReport.getRecords().findAll{it.getType()==DefaultRecordFactory.TYPE_BUILD_RESULT}[0]; +def dependencies = buildReport.getRecords().findAll{it.getType()==DefaultRecordFactory.TYPE_DEPENDENCY_SET}; + +// parse build report to find the build outputs to be deployed. +println("** Find deployable outputs in the build report ") +// the following example finds all the build output with deployType set +//def executes= buildReport.getRecords().findAll{ +// it.getType()==DefaultRecordFactory.TYPE_EXECUTE && +// !it.getOutputs().findAll{ +// o -> o.deployType != null +// }.isEmpty() +//} + +// the following example finds all the build outputs with and without deployType +def executes= buildReport.getRecords().findAll{ + it.getType()==DefaultRecordFactory.TYPE_EXECUTE +} + + +// the following example finds all the build output in *.LOAD data set +//def executes= buildReport.getRecords().findAll{ +// it.getType()==DefaultRecordFactory.TYPE_EXECUTE && +// !it.getOutputs().findAll{ o -> +// def (ds,member) = getDatasetName(o.dataset) +// return ds.endsWith(".LOAD") +// }.isEmpty() +//} + + + +executes.each { it.getOutputs().each { println(" ${it.dataset}, ${it.deployType}")}} + +// generate ship list file. specification of UCD ship list can be found at +// https://www.ibm.com/support/knowledgecenter/SS4GSP_6.2.7/com.ibm.udeploy.doc/topics/zos_shiplistfiles.html +println("** Generate UCD ship list file") +def writer = new StringWriter() +writer.write("\n"); +def xml = new MarkupBuilder(writer) +xml.manifest(type:"MANIFEST_SHIPLIST"){ + //top level property will be added as version properties + //requires UCD v6.2.6 and above + property(name : buildResult.getGroup() + "-" + buildResult.getLabel(), value : buildResult.getUrl()) + //iterate through the outputs and add container and resource elements + executes.each{ execute -> + execute.getOutputs().each{ output -> + def (ds,member) = getDatasetName(output.dataset) + container(name:ds, type:"PDS"){ + resource(name:member, type:"PDSMember", deployType:output.deployType){ + // add any custom properties needed + property(name:"buildcommand", value:execute.getCommand()) + property(name:"buildoptions", value:execute.getOptions()) + if (output.deployType.equals("DBRM")){ + propertyRecord = buildReport.getRecords().findAll{ + it.getType()==DefaultRecordFactory.TYPE_PROPERTIES && it.getProperty("file")==execute.getFile() + } + propertyRecord.each { propertyRec -> + // Iterate Properties + (propertyRec.getProperties()).each { + property(name:"$it.key", value:it.value) + + } + } + } + + // add source information + inputs(url : ""){ + input(name : execute.getFile(), compileType : "Main") + dependencies.each{ + if(it.getId() == execute.getFile()){ + it.getAllDependencies().each{ + def displayName = it.getFile()? it.getFile() : it.getLname() + input(name : displayName, compileType : it.getCategory()) + } + } + } + } + } + } + } + } +} +println("** Write ship list file to $properties.workDir/shiplist.xml") +def shiplistFile = new File("${properties.workDir}/shiplist.xml") +shiplistFile.text = writer + +// assemble and run UCD buztool command to create a version. An example of the command is like below +// /opt/ibm-ucd/agent/bin/buztool.sh createzosversion -c MYCOMP -s /var/dbb/workDir/shiplist.xml +// command parameters can be found at +// https://www.ibm.com/support/knowledgecenter/SS4GSP_6.2.7/com.ibm.udeploy.doc/topics/zos_runtools_uss.html + +def cmd = [ + properties.buztoolPath, + "createzosversion", + "-c", + properties.component, + "-s", + "$properties.workDir/shiplist.xml", + //requires UCD v6.2.6 and above + "-o", + "${properties.workDir}/buztool.output" + +] +// set aritifactRepository option if specified +if (properties.artifactRepositorySettings) { + cmd << "-ar" + cmd << properties.artifactRepositorySettings +} + +//set component version name if specified +if(properties.versionName){ + cmd << "-v" + cmd << properties.versionName +} + +def cmdStr = ""; +cmd.each{ cmdStr = cmdStr + it + " "} +println("** Following UCD buztool cmd will be invoked") +println cmdStr + +// execute command, if no preview is set +if (!properties.preview.toBoolean()){ + + + println("** Create version by running UCD buztool") + def p = cmd.execute(); + p.waitFor(); + println(p.text); + + def rc = p.exitValue(); + if(rc==0){ + println("** buztool output properties") + def outputProp = new Properties() + new File("${properties.workDir}/buztool.output").withInputStream { outputProp.load(it) } + outputProp.each{k,v-> + println " $k -> $v" + } + }else{ + System.exit(rc) + } +} + +/** + * parse data set name and member name + * @param fullname e.g. BLD.LOAD(PGM1) + * @return e.g. (BLD.LOAD, PGM1) + */ +def getDatasetName(String fullname){ + def ds,member; + def elements = fullname.split("[\\(\\)]"); + ds = elements[0]; + member = elements.size()>1? elements[1] : ""; + return [ds, member]; +} + +def parseInput(String[] cliArgs){ + def cli = new CliBuilder(usage: "deploy.groovy [options]") + cli.b(longOpt:'buztool', args:1, argName:'file', 'Absolute path to UrbanCode Deploy buztool.sh script') + cli.w(longOpt:'workDir', args:1, argName:'dir', 'Absolute path to the DBB build output directory') + cli.c(longOpt:'component', args:1, argName:'name', 'Name of the UCD component to create version in') + cli.ar(longOpt:'artifactRepository', args:1, argName:'artifactRepositorySettings', 'Absolute path to Artifactory Server connection file') + cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the UCD component version') + cli.p(longOpt:'preview', 'Preview mode - generate shiplist, but do not run buztool.sh') + cli.h(longOpt:'help', 'Prints this message') + def opts = cli.parse(cliArgs) + if (opts.h) { // if help option used, print usage and exit + cli.usage() + System.exit(0) + } + + def properties = new Properties() + + // load workDir from ./build.properties if it exists + def buildProperties = new Properties() + def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent + def buildPropFile = new File("$scriptDir/build.properties") + if (buildPropFile.exists()){ + buildPropFile.withInputStream { buildProperties.load(it) } + if (buildProperties.workDir != null) + properties.workDir = buildProperties.workDir + } + + // set command line arguments + if (opts.w) properties.workDir = opts.w + if (opts.b) properties.buztoolPath = opts.b + if (opts.c) properties.component = opts.c + if (opts.ar) properties.artifactRepositorySettings = opts.ar + if (opts.v) properties.versionName = opts.v + properties.preview = (opts.p) ? 'true' : 'false' + + // validate required properties + try { + assert properties.buztoolPath : "Missing property buztool script path" + assert properties.workDir: "Missing property build work directory" + assert properties.component: "Missing property UCD component" + } catch (AssertionError e) { + cli.usage() + throw e + } + return properties +} \ No newline at end of file diff --git a/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.md b/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.md new file mode 100644 index 00000000..6532ceed --- /dev/null +++ b/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.md @@ -0,0 +1,44 @@ +# IBM UCD Packaging based on IBM DBB Build Report + +An important step in the pipeline is to generate a deployable package. This sample groovy script +- extracts information about the build outputs from the DBB BuildReport.json +- generates the UCD shiplist ```shiplist.xml``` file +- invokes the ```buztool.sh``` of UCD + +Example invocation: +``` +$DBB_HOME/bin/groovyz dbb-ucd-packaging.groovy --buztool /var/ucd/agent/bin/buztool.sh --workDir /var/ucd/packaging-samples --component MYCOMP --ar /var/ucd/agent/conf/artifactrepository/artifactory.properties --versionName MyVersion +``` + +## Processing flow +- Read command line parameters +- Read DBB's BuildReport.json from the pipeline work directory +- Parse and extract build output information from ExecuteRecords +- Optionally adds generic PropertyRecords, which are linked to the buildfile +- Generates shiplist.xml file +- Invokes buztool.sh on USS with the generated options + +## Command Line Options Summary +``` +$DBB_HOME/bin/groovyz /dbb-ucd-packaging.groovy [options] + +required options: + + -b,--buztool Absolute path to UrbanCode Deploy + buztool.sh script + -c,--component Name of the UCD component to + create version in + -w,--workDir Absolute path to the DBB build + output directory + +optional options: + -ar,--artifactRepository Absolute path to Artifact Respository Server + Server connection file + -v,--versionName Name of the UCD component version + -p,--preview Preview mode generate shiplist, but do + not run buztool.sh + + +utility options + -help,--help Prints this message + ``` \ No newline at end of file From 7a740dad67fcd94fd9736d026a8554157f8e3b0a Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 8 Jun 2020 10:39:34 +0200 Subject: [PATCH 2/4] rename markdown --- .../UCD-Packaging/{dbb-ucd-packaging.md => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Pipeline-Integration/UCD-Packaging/{dbb-ucd-packaging.md => README.md} (100%) diff --git a/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.md b/Pipeline-Integration/UCD-Packaging/README.md similarity index 100% rename from Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.md rename to Pipeline-Integration/UCD-Packaging/README.md From d259d57b03ddcee2b5c6a5aca0ae0d7f55ef460f Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 19 Jun 2020 07:45:52 +0200 Subject: [PATCH 3/4] UCD Packaging Samples --- Pipeline-Integration/README.md | 2 -- .../CreateUCDComponentVersion}/README.md | 0 .../CreateUCDComponentVersion}/dbb-ucd-packaging.groovy | 2 +- Pipeline/README.md | 7 +++++++ 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 Pipeline-Integration/README.md rename {Pipeline-Integration/UCD-Packaging => Pipeline/CreateUCDComponentVersion}/README.md (100%) rename {Pipeline-Integration/UCD-Packaging => Pipeline/CreateUCDComponentVersion}/dbb-ucd-packaging.groovy (99%) create mode 100644 Pipeline/README.md diff --git a/Pipeline-Integration/README.md b/Pipeline-Integration/README.md deleted file mode 100644 index f34c88ee..00000000 --- a/Pipeline-Integration/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Pipeline Integration Samples -This category contains examples of using DBB APIs to integrate additional parts of the pipeline. \ No newline at end of file diff --git a/Pipeline-Integration/UCD-Packaging/README.md b/Pipeline/CreateUCDComponentVersion/README.md similarity index 100% rename from Pipeline-Integration/UCD-Packaging/README.md rename to Pipeline/CreateUCDComponentVersion/README.md diff --git a/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy b/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy similarity index 99% rename from Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy rename to Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy index cf00b1a1..b9bb69fc 100644 --- a/Pipeline-Integration/UCD-Packaging/dbb-ucd-packaging.groovy +++ b/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy @@ -7,7 +7,7 @@ import groovy.xml.MarkupBuilder /** * This script creates a version in UrbanCode Deploy based on the build result. * - * usage: ucd-packaging.groovy [options] + * usage: dbb-ucd-packaging.groovy [options] * * options: * -b,--buztool Absolute path to UrbanCode Deploy buztool.sh script diff --git a/Pipeline/README.md b/Pipeline/README.md new file mode 100644 index 00000000..8e4e72d5 --- /dev/null +++ b/Pipeline/README.md @@ -0,0 +1,7 @@ +# Pipeline Integration Samples +This category contains examples of using DBB APIs to integrate additional parts of the pipeline. + +## Table of Contents +Sample | Description | Documentation Link +--- | --- | --- +CreateUCDComponentVersion | This sample reads the DBB Build Report to generate a UCD Shiplist and to run the BUZTOOL.sh command to create a new UCD Component version. Both Codestation and the external Artifact Repository is supported. | [CreateUCDComponentVersion/README.md](CreateUCDComponentVersion/README.md) From aa1978b372b749f4617982418340b93c55fe8a74 Mon Sep 17 00:00:00 2001 From: Jean-Yves BAUDY Date: Wed, 24 Jun 2020 11:13:37 +0200 Subject: [PATCH 4/4] Exclude output without a deployType --- .../dbb-ucd-packaging.groovy | 29 ++++++------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy b/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy index b9bb69fc..53a83d20 100644 --- a/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy +++ b/Pipeline/CreateUCDComponentVersion/dbb-ucd-packaging.groovy @@ -66,30 +66,19 @@ def dependencies = buildReport.getRecords().findAll{it.getType()==DefaultRecordF // parse build report to find the build outputs to be deployed. println("** Find deployable outputs in the build report ") -// the following example finds all the build output with deployType set -//def executes= buildReport.getRecords().findAll{ -// it.getType()==DefaultRecordFactory.TYPE_EXECUTE && -// !it.getOutputs().findAll{ -// o -> o.deployType != null -// }.isEmpty() -//} // the following example finds all the build outputs with and without deployType -def executes= buildReport.getRecords().findAll{ - it.getType()==DefaultRecordFactory.TYPE_EXECUTE -} - - -// the following example finds all the build output in *.LOAD data set //def executes= buildReport.getRecords().findAll{ -// it.getType()==DefaultRecordFactory.TYPE_EXECUTE && -// !it.getOutputs().findAll{ o -> -// def (ds,member) = getDatasetName(o.dataset) -// return ds.endsWith(".LOAD") -// }.isEmpty() +// it.getType()==DefaultRecordFactory.TYPE_EXECUTE //} - +// the following example finds all the build outputs with a deployType +def executes= buildReport.getRecords().findAll{ + it.getType()==DefaultRecordFactory.TYPE_EXECUTE && + !it.getOutputs().findAll{ o -> + o.deployType != null + }.isEmpty() +} executes.each { it.getOutputs().each { println(" ${it.dataset}, ${it.deployType}")}} @@ -260,4 +249,4 @@ def parseInput(String[] cliArgs){ throw e } return properties -} \ No newline at end of file +}