diff --git a/README.md b/README.md index 390a248e..6fb2f4e9 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ The zAppBuild sample provides the following *language* build scripts by default: * PSBgen.groovy * MFS.groovy * ZunitConfig.groovy +* Transfer.groovy (for transport non-buildable files like JCL or PROC into build libraries and register them as build output) All language scripts both compile and optionally link-edit programs. The language build scripts are intended to be useful out of the box but depending on the complexity of your applications' build requirements, may require modifications to meet your development team's needs. By following the examples used in the existing language build scripts of keeping all application specific references out of the build scripts and instead using configuration properties with strong default values, the zAppBuild sample can continue to be a generic build solution for all of your specific applications. @@ -46,11 +47,16 @@ zAppBuild supports a number of build scenarios: * **Merge Build** - Build only changed programs which will be merged back into the mainBuildBranch by using a triple-dot git diff. * **Scan Source** - Skip the actual building and only scan source files to store dependency data in collection (migration scenario). * **Scan Source + Outputs** - Skip the actual building and only scan source files and existing load modules to dependency data in source and output collection (migration scenario with static linkage scenarios). +* **Build Preview** - Supplemental build option. Process all phases of the supplied build option, but will not execute the commands. A build report and a build result are generated with a specific status that excludes them in subsequent impact build calculations. -Links to additional documentation is provided in the table below. Instructions on invoking a zAppBuild is included in [BUILD.md](BUILD.md) as well as invocation samples for the above mentioned build scenarios including sample console log. +Instructions on invoking a zAppBuild is included in [docs/BUILD.md](docs/BUILD.md) as well as invocation samples for the above mentioned build scenarios including sample console logs. -zAppBuild comes with a set of reporting features. It helps development teams to understand the impact of changed files across multiple applications. Another feature helps to identify conflicts due to concurrent development activities within their application. An overview of these features are documented in [REPORT.md](REPORT.md). +Application-level build properties such as compile and link options can be defined in various ways to set global defaults, application-level overrides or even file level (member-level) overrides. The various supported strategies are documented in [docs/FilePropertyManagement.md](docs/FilePropertyManagement.md). + +zAppBuild comes with a set of reporting features. It helps development teams to understand the impact of changed files across multiple applications. Another feature helps to identify conflicts due to concurrent development activities within their application. An overview of these features are documented in [docs/REPORTS.md](docs/REPORTS.md). + +Links to additional documentation is provided in the table below. ## Repository Legend Folder/File | Description | Documentation Link diff --git a/build-conf/Assembler.properties b/build-conf/Assembler.properties index 587cce71..1d00addf 100644 --- a/build-conf/Assembler.properties +++ b/build-conf/Assembler.properties @@ -18,6 +18,14 @@ assembler_dbrmPDS=${hlq}.DBRM # assembler load data sets assembler_loadPDS=${hlq}.LOAD +# +# assembler sysadata data set +assembler_sysadataPDS=${hlq}.SYSADATA + +# +# output data set for eqalangx side file +assembler_debugPDS=${hlq}.EQALANGX + # # List the data sets that need to be created and their creation options assembler_srcDatasets=${assembler_srcPDS},${assembler_macroPDS},${assembler_objPDS},${assembler_dbrmPDS} @@ -26,8 +34,17 @@ assembler_srcOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(libra assembler_loadDatasets=${assembler_loadPDS} assembler_loadOptions=cyl space(1,1) dsorg(PO) recfm(U) blksize(32760) dsntype(library) +# +# temporary data set allocations assembler_tempOptions=cyl space(5,5) unit(vio) blksize(80) lrecl(80) recfm(f,b) new +# +# sysadata data set creation options +assembler_sysadataOptions=cyl space(5,5) lrecl(32756) dsorg(PO) recfm(v,b) blksize(32760) dsntype(library) + +# eqalangx sidefile data set creation options +assembler_sidefileOptions=cyl dsorg(PO) dsntype(library) space(5,5) unit(vio) blksize(27998) lrecl(1562) recfm(v,b) + # Allocation of SYSMLSD Dataset used for extracting Compile Messages to Remote Error List assembler_compileErrorFeedbackXmlOptions=tracks space(200,40) dsorg(PS) blksize(27998) lrecl(16383) recfm(v,b) new keep @@ -38,6 +55,7 @@ assembler_outputDatasets=${assembler_loadPDS},${assembler_dbrmPDS} # default assembler execs assembler_db2precompiler=DSNHPC assember_cicsprecompiler=DFHEAP1$ +assembler_eqalangx=EQALANGX assembler_pgm=ASMA90 assembler_linkEditor=IEWBLINK diff --git a/build-conf/README.md b/build-conf/README.md index aa82a0e8..345faf25 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -76,12 +76,16 @@ assembler_srcPDS | Dataset to move assembler source files to from USS assembler_macroPDS | Dataset to move macro files to from USS assembler_objPDS | Dataset to create object decks in from Assembler step assembler_dbrmPDS | Dataset to create DB2 DBRM modules in from Assembler step +assembler_sysadataPDS | Dataset to create sysadata file that contains source and symbolic data about the program +assembler_debugPDS | Dataset to create the eqalangx side file for the IBM Debug Tool assembler_loadPDS | Dataset to create load modules in from link edit step assembler_srcDataSets | Comma separated list of 'source' type data sets assembler_srcOptions | BPXWDYN creation options for creating 'source' type data sets assembler_loadDatasets | Comma separated list of 'load module' type data sets assembler_loadOptions | BPXWDYN creation options for 'load module' type data sets assembler_tempOptions | BPXWDYN creation options for temporary data sets +assembler_sysadataOptions | BPXWDYN creation options for assembler sysadata data sets +assembler_sidefileOptions | BPXWDYN creation options for eqalangx sidefile assembler_compileErrorFeedbackXmlOptions | BPXWDYN creation options for SYSXMLSD data set assembler_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. assembler_pgm | MVS program name of the high level assembler diff --git a/build-conf/Transfer.properties b/build-conf/Transfer.properties index ca54ce41..fd0ae198 100644 --- a/build-conf/Transfer.properties +++ b/build-conf/Transfer.properties @@ -1,15 +1,15 @@ # Releng properties used by language/Transfer.groovy # Comma separated list of required build properties for Cobol.groovy -transfer_requiredBuildProperties=transfer_srcPDS,transfer_srcOptions,\ +transfer_requiredBuildProperties=transfer_srcPDS,transfer_dsOptions,\ transfer_deployType # -# transfer source data sets +# Transfer source data sets # Add additional dataset definitions, depending on your requirements # # Please note, that files in the repository require to be mapped by a PropertyMapping in file.properties -# to one of the dataset definitions +# to one of the dataset definitions. See samples/application-conf/file.properties # transfer_srcPDS=${hlq}.SOURCE transfer_jclPDS=${hlq}.JCL @@ -17,7 +17,9 @@ transfer_xmlPDS=${hlq}.XML # # dataset creation options -transfer_srcOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library) - -# List of output datasets to document deletions -transfer_outputDatasets=${transfer_srcPDS} \ No newline at end of file +# +# This is using the DBB PropertyMapping syntax allowing multiple mappings of +# target source dataset definitions to the required options +# +transfer_dsOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_srcPDS, transfer_jclPDS +transfer_dsOptions=cyl space(1,1) lrecl(144) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_xmlPDS diff --git a/build-conf/datasets.properties b/build-conf/datasets.properties index 48f5f582..28b407e3 100644 --- a/build-conf/datasets.properties +++ b/build-conf/datasets.properties @@ -71,3 +71,6 @@ SBZUSAMP= # REXX Compiler Data Sets. Example: REXX.V1R4.SFANLMD SFANLMD= + +# PD Tools Common Component load library. Example : PDTCC.V1R8.SIPVMODA +PDTCCMOD= diff --git a/build.groovy b/build.groovy index 91fea08e..609dcb90 100644 --- a/build.groovy +++ b/build.groovy @@ -17,6 +17,7 @@ import groovy.cli.commons.* @Field def gitUtils= loadScript(new File("utilities/GitUtilities.groovy")) @Field def buildUtils= loadScript(new File("utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("utilities/ImpactUtilities.groovy")) +@Field def reportingUtils= loadScript(new File("utilities/ReportingUtilities.groovy")) @Field def filePropUtils= loadScript(new File("utilities/FilePropUtilities.groovy")) @Field String hashPrefix = ':githash:' @Field String giturlPrefix = ':giturl:' @@ -104,6 +105,12 @@ def initializeBuildProcess(String[] args) { // verify required build properties buildUtils.assertBuildProperties(props.requiredBuildProperties) + // evaluate preview flag to set the reportOnly + if (props.preview) { + println "** Running in reportOnly mode. Will process build options but not execute any steps." + props.put("dbb.command.reportOnly","true") + } + // create metadata store for this script if (!props.userBuild) { if (props.metadataStoreType == 'file') @@ -190,13 +197,18 @@ def initializeBuildProcess(String[] args) { // initialize build result (requires MetadataStore) if (metadataStore) { def buildResult = metadataStore.createBuildResult(props.applicationBuildGroup, props.applicationBuildLabel) + // set build state and status buildResult.setState(buildResult.PROCESSING) + if (props.preview) buildResult.setStatus(4) + if (props.scanOnly) buildResult.setProperty('scanOnly', 'true') if (props.fullBuild) buildResult.setProperty('fullBuild', 'true') if (props.impactBuild) buildResult.setProperty('impactBuild', 'true') if (props.topicBranchBuild) buildResult.setProperty('topicBranchBuild', 'true') + if (props.preview) buildResult.setProperty('preview', 'true') + if (props.buildFile) buildResult.setProperty('buildFile', XmlUtil.escapeXml(props.buildFile)) - + println("** Build result created for BuildGroup:${props.applicationBuildGroup} BuildLabel:${props.applicationBuildLabel}") } @@ -231,7 +243,8 @@ options: cli.m(longOpt:'mergeBuild', 'Flag indicating to build only changes which will be merged back to the mainBuildBranch.') cli.r(longOpt:'reset', 'Deletes the dependency collections and build result group from the MetadataStore') cli.v(longOpt:'verbose', 'Flag to turn on script trace') - + cli.pv(longOpt:'preview', 'Supplemental flag indicating to run build in preview mode without processing the execute commands') + // scan options cli.s(longOpt:'scanOnly', 'Flag indicating to only scan source files for application without building anything (deprecated use --scanSource)') cli.ss(longOpt:'scanSource', 'Flag indicating to only scan source files for application without building anything') @@ -392,7 +405,8 @@ def populateBuildProperties(def opts) { if (opts.v) props.verbose = 'true' if (opts.b) props.baselineRef = opts.b if (opts.m) props.mergeBuild = 'true' - + if (opts.pv) props.preview = 'true' + // scan options if (opts.s) props.scanOnly = 'true' if (opts.ss) props.scanOnly = 'true' @@ -492,9 +506,13 @@ def createBuildList() { Set changedBuildProperties = new HashSet() // not yet used for any post-processing String action = (props.scanOnly) || (props.scanLoadmodules) ? 'Scanning' : 'Building' + // check if preview sub-option + if (props.preview) { println "** --preview cli option provided. Processing all phases of the supplied build option, but will not execute the commands." } + // check if full build if (props.fullBuild) { println "** --fullBuild option selected. $action all programs for application ${props.application}" + buildSet = buildUtils.createFullBuildList() } // check if impact build @@ -515,7 +533,7 @@ def createBuildList() { println "*! Merge build requires a Filesystem or Db2 MetadataStore" } } - + // if build file present add additional files to build list (mandatory build list) if (props.buildFile) { @@ -596,18 +614,18 @@ def createBuildList() { if (props.reportExternalImpacts && props.reportExternalImpacts.toBoolean()){ if (buildSet && changedFiles) { println "** Perform analysis and reporting of external impacted files for the build list including changed files." - impactUtils.reportExternalImpacts(buildSet.plus(changedFiles)) + reportingUtils.reportExternalImpacts(buildSet.plus(changedFiles)) } else if(buildSet) { println "** Perform analysis and reporting of external impacted files for the build list." - impactUtils.reportExternalImpacts(buildSet) + reportingUtils.reportExternalImpacts(buildSet) } } // Document and validate concurrent changes if (props.reportConcurrentChanges && props.reportConcurrentChanges.toBoolean()){ println "** Calculate and document concurrent changes." - impactUtils.calculateConcurrentChanges(buildSet) + reportingUtils.calculateConcurrentChanges(buildSet) } // document deletions in build report @@ -675,7 +693,6 @@ def finalizeBuildProcess(Map args) { buildResult.setState(buildResult.COMPLETE) - // store build result properties in BuildReport.json PropertiesRecord buildReportRecord = new PropertiesRecord("DBB.BuildResultProperties") def buildResultProps = buildResult.getPropertyNames() @@ -688,7 +705,7 @@ def finalizeBuildProcess(Map args) { // } buildReport.addRecord(buildReportRecord) } - + // create build report data file def jsonOutputFile = new File("${props.buildOutDir}/BuildReport.json") def buildReportEncoding = "UTF-8" @@ -719,6 +736,7 @@ def finalizeBuildProcess(Map args) { def state = (props.error) ? "ERROR" : "CLEAN" println("** Build ended at $endTime") println("** Build State : $state") + if (props.preview) println("** Build ran in preview mode.") println("** Total files processed : ${args.count}") println("** Total build time : $duration\n") } diff --git a/BUILD.md b/docs/BUILD.md similarity index 84% rename from BUILD.md rename to docs/BUILD.md index b350a928..6980ca39 100644 --- a/BUILD.md +++ b/docs/BUILD.md @@ -200,7 +200,9 @@ build options: -ss,--scanSource Flag indicating to only scan source files for application without building anything -sl,--scanLoad Flag indicating to only scan load modules for application without building anything -sa,--scanAll Flag indicating to scan both source files and load modules for application without building anything - + -pv,--preview Supplemental flag indicating to run build in preview mode without processing the execute commands + + -r,--reset Deletes the application's dependency collections and build result group from the DBB repository -v,--verbose Flag to turn on script trace @@ -250,6 +252,7 @@ utility options - [Perform Impact Build for topic branches](#perform-impact-build-for-topic-branches) - [Perform Impact Build by providing baseline reference for the analysis of changed files](#perform-impact-build-by-providing-baseline-reference-for-the-analysis-of-changed-files) - [Perform a Merge build](#perform-a-merge-build) +- [Perform a Build in preview mode](#perform-a-build-in-preview-mode) - [Perform a Scan Source build](#perform-a-scan-source-build) - [Perform a Scan Source + Outputs build](#perform-a-scan-source--outputs-build) - [Dynamically Overwrite build properties](#dynamically-overwrite-build-properties) @@ -1198,6 +1201,355 @@ Cobol compiler parms for MortgageApplication/cobol/epsmlist.cbl = LIB,CICS ``` + + +### Perform a Build in Preview Mode + +`--preview` is a supplemental option to the various build types of zAppBuild. This supplemental option will let the build process run through all the build phases of the specified build option, but instead of executing the build commands such as copying the source files to the datasets or invoking the compile and link commands, it just documents what would be done and sets the return codes of these commands to 0. Please note, that file are scanned and, depending on the primary build option, dependency information is stored in the DBB Metadatastore. + +For instance, use the `--preview` flag with the `--impactBuild` option to obtain a preview of the impact build actions such as identified changed files, the calculated impacted files, the build list, the build flow, the applied build properties and option including the outputs which would be produced. + +Use the `--preview` flag with the `--fullBuild` option to produce the full bill of material (documented in a build report) for the artifacts that could be generated in the datasets pointed by the `hlq` parameter. + +The build will generate a build report, which, depending of the provided build option, will be stored in the build group. However, the build result status is set to `4` and does not impact the calculation of changed file of subsequent impact builds. + +The below sample build log is documenting an `--impactBuild --preview` with the reporting capablities activated to what the build would do and any potential conflicts of concurrent development activities. + +``` +groovyz dbb-zappbuild/build.groovy --workspace /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --mergeBuild --verbose +``` +
+ Build log + +``` ++ /usr/lpp/dbb/v2r0/bin/groovyz /var/dbb/dbb-zappbuild/build.groovy --sourceDir /var/dbb/dbb-zappbuild/samples --workDir /var/dbb/work/mortgageout --url jdbc:db2://10.3.20.201:4740/MOPDBC0 --id DB2ID --pwFile /var/dbb/config/db2-pwd-file.xml --hlq DBEHM.DBB.BUILD --application MortgageApplication --verbose --impactBuild --preview --propFiles /var/dbb/dbb-zappbuild-config/build.properties,/var/dbb/dbb-zappbuild-config/datasets.properties + +** Build start at 20230425.040722.007 +** Input args = /var/dbb/dbb-zappbuild/samples --workDir /var/dbb/work/mortgageout --url jdbc:db2://10.3.20.201:4740/MOPDBC0 --id DB2ID --pwFile /var/dbb/config/db2-pwd-file.xml --hlq DBEHM.DBB.BUILD --application MortgageApplication --verbose --impactBuild --preview --propFiles /var/dbb/dbb-zappbuild-config/build.properties,/var/dbb/dbb-zappbuild-config/datasets.properties,/var/dbb/dbb-zappbuild-config/dbehm.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/datasets.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/dependencyReport.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/Assembler.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/BMS.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/MFS.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/PSBgen.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/DBDgen.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/ACBgen.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/Cobol.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/LinkEdit.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/PLI.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/REXX.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/ZunitConfig.properties +** Loading property file /var/dbb/dbb-zappbuild/build-conf/Transfer.properties +** appConf = /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/file.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/BMS.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/Cobol.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/LinkEdit.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/languageConfigurationMapping.properties +java.version=8.0.7.20 - pmz6480sr7fp20-20221020_01(SR7 FP20) +java.home=/V2R4/usr/lpp/java/J8.0_64 +user.dir=/u/builduser +** Build properties at start up: +... +preview=true +... +** zAppBuild running on DBB Toolkit Version 2.0.0 20-Mar-2023 10:36:28 +required props = buildOrder,buildListFileExt +** Running in reportOnly mode. Will process build options but not execute any steps. +** Db2 MetadataStore initialized +** Build output located at /var/dbb/work/mortgageout/build.20230425.160722.007 +** Build result created for BuildGroup:MortgageApplication-350_preview_builds BuildLabel:build.20230425.160722.007 +** --preview cli option provided. Processing all phases of the supplied build option, but will not execute the commands. +** --impactBuild option selected. Building impacted programs for application MortgageApplication +** Getting current hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Storing MortgageApplication : 2b3add1e85a8124ff1d7af6ab1de2e5463325d7a +** Getting baseline hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Storing MortgageApplication : cf6bc732bd717b404c5cf71a8f8d14458138a2d0 +** Calculating changed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Diffing baseline cf6bc732bd717b404c5cf71a8f8d14458138a2d0 -> current 2b3add1e85a8124ff1d7af6ab1de2e5463325d7a +*** Changed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication : +**** MortgageApplication/jcl/MYSAMP.jcl +**** MortgageApplication/copybook/epsmtcom.cpy +*** Deleted files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication : +*** Renamed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication : +** Updating collections MortgageApplication-350_preview_builds and MortgageApplication-350_preview_builds-outputs +*** Sorted list of changed files: [MortgageApplication/jcl/MYSAMP.jcl, MortgageApplication/copybook/epsmtcom.cpy] +*** Scanning file MortgageApplication/jcl/MYSAMP.jcl (/var/dbb/dbb-zappbuild/samples/MortgageApplication/copybook/MYSAMP.jcl) +*** Scanning file with the default scanner +*** Logical file for MortgageApplication/copybook/MYSAMP.jcl = +{ + "cics": false, + "dli": false, + "file": "MortgageApplication\/jcl\/MYSAMP.jcl", + "language": "JCL", + "lname": "MYSAMP", + "logicalDependencies": [ + { + "category": "COPY", + "library": "SYSLIB", + "lname": "EPSMTINP" + }, + { + "category": "COPY", + "library": "SYSLIB", + "lname": "EPSMTOUT" + } + ], + "mq": false, + "sql": false +} +*** Scanning file MortgageApplication/copybook/epsmtcom.cpy (/var/dbb/dbb-zappbuild/samples/MortgageApplication/copybook/epsmtcom.cpy) +*** Scanning file with the default scanner +*** Logical file for MortgageApplication/copybook/epsmtcom.cpy = +{ + "cics": false, + "dli": false, + "file": "MortgageApplication\/copybook\/epsmtcom.cpy", + "language": "COB", + "lname": "EPSMTCOM", + "logicalDependencies": [ + { + "category": "COPY", + "library": "SYSLIB", + "lname": "EPSMTINP" + }, + { + "category": "COPY", + "library": "SYSLIB", + "lname": "EPSMTOUT" + } + ], + "mq": false, + "sql": false +} +** Storing 2 logical files in MetadataStore collection 'MortgageApplication-350_preview_builds' +*** Perform impacted analysis for changed files. +** Found build script mapping for MortgageApplication/jcl/MYSAMP.jcl. Adding to build list +** Performing impact analysis on changed file MortgageApplication/jcl/MYSAMP.jcl +*** Creating SearchPathImpactFinder with collections [MortgageApplication-350_preview_builds, MortgageApplication-350_preview_builds-outputs] and impactSearch configuration search:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/copybook/*.cpysearch:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/bms/*.bmssearch:[:LINK]/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/cobol/*.cbl +** Performing impact analysis on changed file MortgageApplication/copybook/epsmtcom.cpy +*** Creating SearchPathImpactFinder with collections [MortgageApplication-350_preview_builds, MortgageApplication-350_preview_builds-outputs] and impactSearch configuration search:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/copybook/*.cpysearch:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/bms/*.bmssearch:[:LINK]/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/cobol/*.cbl +** Found impacted file MortgageApplication/cobol/epscmort.cbl +** MortgageApplication/cobol/epscmort.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epscsmrt.cbl +** MortgageApplication/cobol/epscsmrt.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epsmlist.cbl +** MortgageApplication/cobol/epsmlist.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epscsmrt.cbl +** MortgageApplication/cobol/epscsmrt.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epscmort.cbl +** MortgageApplication/cobol/epscmort.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/link/epsmlist.lnk +** MortgageApplication/link/epsmlist.lnk is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Calculation of impacted files by changed properties has been skipped due to configuration. +** Writing build list file to /var/dbb/work/mortgageout/build.20230425.160722.007/buildList.txt +MortgageApplication/cobol/epsmlist.cbl +MortgageApplication/cobol/epscsmrt.cbl +MortgageApplication/cobol/epscmort.cbl +MortgageApplication/link/epsmlist.lnk +MortgageApplication/jcl/MYSAMP.jcl +** Populating file level properties from individual artifact properties files. +* Populating file level properties overrides. +** Checking file property overrides for MortgageApplication/cobol/epsmlist.cbl +*** MortgageApplication/cobol/epsmlist.cbl has an individual artifact properties file defined in properties/epsmlist.cbl.properties + Found file property cobol_compileParms = ${cobol_compileParms},SOURCE +*** Checking for existing file property overrides + Checking build property cobol_compileParms + Adding file property override cobol_compileParms = ${cobol_compileParms},SOURCE for MortgageApplication/cobol/epsmlist.cbl +** Checking file property overrides for MortgageApplication/cobol/epscsmrt.cbl +*** Checking for existing file property overrides +** Checking file property overrides for MortgageApplication/cobol/epscmort.cbl +*** Checking for existing file property overrides +** Checking file property overrides for MortgageApplication/link/epsmlist.lnk +*** Checking for existing file property overrides +** Checking file property overrides for MortgageApplication/jcl/MYSAMP.jcl +*** Checking for existing file property overrides +** Perform analysis and reporting of external impacted files for the build list including changed files. +*** Running external impact analysis with file filter **/* and collection patterns .*-main.* with analysis mode deep +*** Running external impact analysis for files + MortgageApplication/cobol/epscmort.cbl + MortgageApplication/cobol/epsmlist.cbl + MortgageApplication/link/epsmlist.lnk + MortgageApplication/jcl/MYSAMP.jcl + MortgageApplication/cobol/epscsmrt.cbl + MortgageApplication/copybook/epsmtcom.cpy +*** Writing report of external impacts to file /var/dbb/work/mortgageout/build.20230425.160722.007/externalImpacts_MortgageApplication-main-outputs.log +*** Potential external impact found EPSCSMRT (MortgageApplication/cobol/epscsmrt.cbl) in collection MortgageApplication-main-outputs +*** Potential external impact found EPSCMORT (MortgageApplication/cobol/epscmort.cbl) in collection MortgageApplication-main-outputs +*** Potential external impact found EPSMLIST (MortgageApplication/link/epsmlist.lnk) in collection MortgageApplication-main-outputs +*** Writing report of external impacts to file /var/dbb/work/mortgageout/build.20230425.160722.007/externalImpacts_MortgageApplication-main-patch.log +*** Potential external impact found EPSCSMRT (MortgageApplication/cobol/epscsmrt.cbl) in collection MortgageApplication-main-patch +*** Potential external impact found EPSMLIST (MortgageApplication/cobol/epsmlist.cbl) in collection MortgageApplication-main-patch +*** Potential external impact found EPSCMORT (MortgageApplication/cobol/epscmort.cbl) in collection MortgageApplication-main-patch +*** Writing report of external impacts to file /var/dbb/work/mortgageout/build.20230425.160722.007/externalImpacts_MortgageApplication-main.log +*** Potential external impact found EPSCSMRT (MortgageApplication/cobol/epscsmrt.cbl) in collection MortgageApplication-main +*** Potential external impact found EPSMLIST (MortgageApplication/cobol/epsmlist.cbl) in collection MortgageApplication-main +*** Potential external impact found EPSCMORT (MortgageApplication/cobol/epscmort.cbl) in collection MortgageApplication-main +*** Writing report of external impacts to file /var/dbb/work/mortgageout/build.20230425.160722.007/externalImpacts_MortgageApplication-main-patch-outputs.log +*** Potential external impact found EPSMLIST (MortgageApplication/link/epsmlist.lnk) in collection MortgageApplication-main-patch-outputs +**** Running external impact analysis for identified external impacted files as dependent files of the initial set. + MortgageApplication/cobol/epscsmrt.cbl + MortgageApplication/cobol/epscmort.cbl + MortgageApplication/link/epsmlist.lnk + MortgageApplication/cobol/epscsmrt.cbl + MortgageApplication/cobol/epsmlist.cbl + MortgageApplication/cobol/epscmort.cbl + MortgageApplication/cobol/epscsmrt.cbl + MortgageApplication/cobol/epsmlist.cbl + MortgageApplication/cobol/epscmort.cbl + MortgageApplication/link/epsmlist.lnk +** Calculate and document concurrent changes. +*** Analysing and validating changes for branch : main +** Getting current hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Storing MortgageApplication : 2b3add1e85a8124ff1d7af6ab1de2e5463325d7a +** Calculating changed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +*** Changed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication in configuration main: +**** MortgageApplication/cobol/epscmort.cbl +*** Deleted files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication in configuration main: +*** Renamed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication in configuration main: +** Writing report of concurrent changes to /var/dbb/work/mortgageout/build.20230425.160722.007/report_concurrentChanges.txt for configuration main + Changed: MortgageApplication/cobol/epscmort.cbl +*!! MortgageApplication/cobol/epscmort.cbl is changed on branch main and intersects with the current build list. +** Invoking build scripts according to build order: BMS.groovy,Cobol.groovy,LinkEdit.groovy,Transfer.groovy +** Building files mapped to Cobol.groovy script +required props = cobol_srcPDS,cobol_cpyPDS,cobol_objPDS,cobol_loadPDS,cobol_compiler,cobol_linkEditor,cobol_tempOptions,applicationOutputsCollectionName,SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED, cobol_dependencySearch +** Creating / verifying build dataset DBEHM.DBB.BUILD.COBOL +** Creating / verifying build dataset DBEHM.DBB.BUILD.COPY +** Creating / verifying build dataset DBEHM.DBB.BUILD.OBJ +** Creating / verifying build dataset DBEHM.DBB.BUILD.DBRM +** Creating / verifying build dataset DBEHM.DBB.BUILD.LOAD +*** Building file MortgageApplication/cobol/epsmlist.cbl +*** Resolution rules for MortgageApplication/cobol/epsmlist.cbl: +search:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/copybook/*.cpy +*** Physical dependencies for MortgageApplication/cobol/epsmlist.cbl: +{"excluded":false,"lname":"DFHAID","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"lname":"EPSMLIS","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMORTF","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmortf.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTCOM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtcom.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTINP","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtinp.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTOUT","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtout.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSNBRPM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsnbrpm.cpy","category":"COPY","resolved":true} +Program attributes: CICS=true, SQL=false, DLI=false, MQ=false +Cobol compiler parms for MortgageApplication/cobol/epsmlist.cbl = LIB,SOURCE,CICS +Link-Edit parms for MortgageApplication/cobol/epsmlist.cbl = MAP,RENT,COMPAT(PM5),SSI=2b3add1e +*** Building file MortgageApplication/cobol/epscsmrt.cbl +*** Resolution rules for MortgageApplication/cobol/epscsmrt.cbl: +search:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/copybook/*.cpy +*** Physical dependencies for MortgageApplication/cobol/epscsmrt.cbl: +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTCOM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtcom.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTINP","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtinp.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTOUT","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtout.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSPDATA","library":"SYSLIB","file":"MortgageApplication\/copybook\/epspdata.cpy","category":"COPY","resolved":true} +Program attributes: CICS=true*, SQL=false, DLI=false, MQ=false +Cobol compiler parms for MortgageApplication/cobol/epscsmrt.cbl = LIB,CICS +Link-Edit parms for MortgageApplication/cobol/epscsmrt.cbl = MAP,RENT,COMPAT(PM5),SSI=2b3add1e +*** Scanning load module for MortgageApplication/cobol/epscsmrt.cbl +*** Logical file = +{ + "cics": false, + "dli": false, + "file": "MortgageApplication\/cobol\/epscsmrt.cbl", + "language": "ZBND", + "lname": "EPSCSMRT", + "logicalDependencies": [ + { + "category": "LINK", + "library": "DBEHM.DBB.BUILD.OBJ", + "lname": "EPSCSMRT" + } + ], + "mq": false, + "sql": false +} +*** Building file MortgageApplication/cobol/epscmort.cbl +*** Resolution rules for MortgageApplication/cobol/epscmort.cbl: +search:/var/dbb/dbb-zappbuild/samples/?path=MortgageApplication/copybook/*.cpy +*** Physical dependencies for MortgageApplication/cobol/epscmort.cbl: +{"excluded":false,"lname":"DFHAID","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"lname":"EPSMORT","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTCOM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtcom.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTINP","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtinp.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSMTOUT","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtout.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/u\/dbehm\/test-zapp\/dbb-zappbuild\/samples\/","lname":"EPSNBRPM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsnbrpm.cpy","category":"COPY","resolved":true} +{"excluded":false,"lname":"SQLCA","library":"SYSLIB","category":"SQL INCLUDE","resolved":false} +Program attributes: CICS=true, SQL=true, DLI=false, MQ=false +Cobol compiler parms for MortgageApplication/cobol/epscmort.cbl = LIB,CICS,SQL +Link-Edit parms for MortgageApplication/cobol/epscmort.cbl = MAP,RENT,COMPAT(PM5),SSI=2b3add1e +*** Scanning load module for MortgageApplication/cobol/epscmort.cbl +*** Logical file = +{ + "cics": false, + "dli": false, + "file": "MortgageApplication\/cobol\/epscmort.cbl", + "language": "ZBND", + "lname": "EPSCMORT", + "logicalDependencies": [ + { + "category": "LINK", + "library": "DBEHM.DBB.BUILD.OBJ", + "lname": "EPSCMORT" + }, + { + "category": "LINK", + "library": "DBEHM.DBB.BUILD.OBJ", + "lname": "EPSNBRVL" + } + ], + "mq": false, + "sql": false +} +** Building files mapped to LinkEdit.groovy script +required props = linkedit_srcPDS,linkedit_objPDS,linkedit_loadPDS,linkedit_linkEditor,linkedit_tempOptions,applicationOutputsCollectionName, SDFHLOAD,SCEELKED +** Creating / verifying build dataset DBEHM.DBB.BUILD.LINK +** Creating / verifying build dataset DBEHM.DBB.BUILD.OBJ +** Creating / verifying build dataset DBEHM.DBB.BUILD.LOAD +*** Building file MortgageApplication/link/epsmlist.lnk +Link-Edit parms for MortgageApplication/link/epsmlist.lnk = MAP,RENT,COMPAT(PM5),SSI=2b3add1e +*** Scanning load module for MortgageApplication/link/epsmlist.lnk +*** Logical file = +{ + "cics": false, + "dli": false, + "file": "MortgageApplication\/link\/epsmlist.lnk", + "language": "ZBND", + "lname": "EPSMLIST", + "logicalDependencies": [ + { + "category": "LINK", + "library": "DBEHM.DBB.BUILD.LOAD", + "lname": "EPSMPMT" + }, + { + "category": "LINK", + "library": "DBEHM.DBB.BUILD.OBJ", + "lname": "EPSMLIST" + } + ], + "mq": false, + "sql": false +} +** Building files mapped to Transfer.groovy script +required props = transfer_srcPDS,transfer_dsOptions, transfer_deployType +*** Transferring file MortgageApplication/jcl/MYSAMP.jcl +** Creating / verifying build dataset DBEHM.DBB.BUILD.JCL +** Copied MortgageApplication/jcl/MYSAMP.jcl to DBEHM.DBB.BUILD.JCL with deployType JCL (rc = 0) +*** Obtaining hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Setting property :githash:MortgageApplication : 2b3add1e85a8124ff1d7af6ab1de2e5463325d7a +** Setting property :giturl:MortgageApplication : https://github.com/dennis-behm/dbb-zappbuild.git +** Setting property :gitchangedfiles:MortgageApplication : https://github.com/ibm/dbb-zappbuild/compare/cf6bc732bd717b404c5cf71a8f8d14458138a2d0..2b3add1e85a8124ff1d7af6ab1de2e5463325d7a +** Writing build report data to /var/dbb/work/mortgageout/build.20230425.160722.007/BuildReport.json +** Writing build report to /var/dbb/work/mortgageout/build.20230425.160722.007/BuildReport.html +** Updating build result BuildGroup:MortgageApplication-350_preview_builds BuildLabel:build.20230425.160722.007 +** Build ended at Tue Apr 25 16:07:34 GMT+01:00 2023 +** Build State : CLEAN +** Build ran in preview mode. +** Total files processed : 5 +** Total build time : 12.204 seconds + +** Build finished +``` + +
### Perform a Scan Source build @@ -1657,3 +2009,5 @@ required props = linkedit_srcPDS,linkedit_objPDS,linkedit_loadPDS,linkedit_linkE ``` + + diff --git a/docs/README.md b/docs/README.md index 8ef04565..7982ca0d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,4 +4,7 @@ This folder contains supplemental documentation to help explain the implementati |File|Description| |-|-| -|FilePropertyManagement|Documentation on managing default and overriding build properties for specific application files| +|[Build Management](BUILD.md)|Documentation of the main and start build script `build.groovy`, reference of the command line options of the build script and documentation of the build scenarios supported by zAppBuild | +|[Build Property Management](FilePropertyManagement.md)|Documentation on managing default and overriding build properties for specific application files| +|[Reporting Capabilities](REPORTS.md)|Documentation of zAppBuild reporting capabilities like reporting external impacted file and reporting of concurrent changes| + diff --git a/REPORT.md b/docs/REPORTS.md similarity index 97% rename from REPORT.md rename to docs/REPORTS.md index 01a39d4e..b562f3ba 100644 --- a/REPORT.md +++ b/docs/REPORTS.md @@ -110,7 +110,7 @@ App-EPSM/cobol/epsplist.cbl ### Purpose -Concurrent development activies in separated isolated branches on the same source code, lead to the need to merge the code at some point in time. Git does an excellent job for this task and supports the developer for this task. While pessimistic locking is a common practise on the mainframe, developers will need to keep an eye on what is happening in the git repository, which follows the optimistic locking approach. +Concurrent development activities in separated isolated branches on the same source code, lead to the need to merge the code at some point in time. Git does an excellent job for this task and supports the developer for this task. While pessimistic locking is a common practise on the mainframe, developers will need to keep an eye on what is happening in the git repository, which follows the optimistic locking approach. The _Report concurrent changes_ feature can be activated to generate a report to document changes in other configurations within the repository. Additionally, it validates if the current build list intersects with those changes. diff --git a/languages/Assembler.groovy b/languages/Assembler.groovy index dc329ffe..f6dbbcd6 100644 --- a/languages/Assembler.groovy +++ b/languages/Assembler.groovy @@ -12,7 +12,7 @@ import com.ibm.dbb.build.report.records.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.assembler_requiredBuildProperties) @@ -20,12 +20,23 @@ buildUtils.assertBuildProperties(props.assembler_requiredBuildProperties) def langQualifier = "assembler" buildUtils.createLanguageDatasets(langQualifier) +// create sysadata dataset used in errorPrefix and debug +if (props.errPrefix || props.debug) { + buildUtils.createDatasets(props.assembler_sysadataPDS.split(), props.assembler_sysadataOptions) +} + +// create debug dataset for the sidefile +if (props.debug) { + buildUtils.createDatasets(props.assembler_debugPDS.split(), props.assembler_sidefileOptions) +} + // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'assembler_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'assembler_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // Configure dependency resolution String dependencySearch = props.getFileProperty('assembler_dependencySearch', buildFile) @@ -37,6 +48,9 @@ sortedList.each { buildFile -> // Create logical file LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) + // print logicalFile details and overrides + if (props.verbose) buildUtils.printLogicalFileAttributes(logicalFile) + // create mvs commands String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.asm.log") @@ -45,6 +59,7 @@ sortedList.each { buildFile -> MVSExec assembler_SQLTranslator = createAssemblerSQLTranslatorCommand(buildFile, logicalFile, member, logFile) MVSExec assembler_CICSTranslator = createAssemblerCICSTranslatorCommand(buildFile, logicalFile, member, logFile) MVSExec assembler = createAssemblerCommand(buildFile, logicalFile, member, logFile) + MVSExec debugSideFile = createDebugSideFile(buildFile, logicalFile, member, logFile) MVSExec linkEdit = createLinkEditCommand(buildFile, logicalFile, member, logFile) // execute mvs commands in a mvs job @@ -53,11 +68,14 @@ sortedList.each { buildFile -> // initialize return codes int rc = 0 + int maxRC = props.getFileProperty('assembler_maxRC', buildFile).toInteger() // SQL preprocessor if (buildUtils.isSQL(logicalFile)){ rc = assembler_SQLTranslator.execute() + maxRC = props.getFileProperty('assembler_maxSQLTranslatorRC', buildFile).toInteger() + if (rc > maxRC) { String errorMsg = "*! The assembler sql translator return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" println(errorMsg) @@ -74,8 +92,10 @@ sortedList.each { buildFile -> } // CICS preprocessor - if (buildUtils.isCICS(logicalFile)){ + if (rc <= maxRC && buildUtils.isCICS(logicalFile)){ rc = assembler_CICSTranslator.execute() + maxRC = props.getFileProperty('assembler_maxCICSTranslatorRC', buildFile).toInteger() + if (rc > maxRC) { String errorMsg = "*! The assembler cics translator return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" println(errorMsg) @@ -95,34 +115,47 @@ sortedList.each { buildFile -> props.error = "true" buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile]) } + } + + // create sidefile + if (rc <= maxRC && props.debug) { + rc = debugSideFile.execute() + maxRC = props.getFileProperty('assembler_maxIDILANGX_RC', buildFile).toInteger() + + if (rc > maxRC) { + String errorMsg = "*! The preparation step of the sidefile EQALANX return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile]) + } + } + + + String needsLinking = props.getFileProperty('assembler_linkEdit', buildFile) + // linkedit + if (rc <= maxRC && needsLinking && needsLinking.toBoolean()) { + + rc = linkEdit.execute() + maxRC = props.getFileProperty('assembler_linkEditMaxRC', buildFile).toInteger() + + if (rc > maxRC) { + String errorMsg = "*! The link edit return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile]) + } else { - // if this program needs to be link edited . . . - String needsLinking = props.getFileProperty('assembler_linkEdit', buildFile) - if (needsLinking && needsLinking.toBoolean()) { - rc = linkEdit.execute() - maxRC = props.getFileProperty('assembler_linkEditMaxRC', buildFile).toInteger() - - if (rc > maxRC) { - String errorMsg = "*! The link edit return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)" - println(errorMsg) - props.error = "true" - buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile]) - } - else { - // only scan the load module if load module scanning turned on for file - if(!props.userBuild){ - String scanLoadModule = props.getFileProperty('assembler_scanLoadModule', buildFile) - if (scanLoadModule && scanLoadModule.toBoolean()) { - String assembler_loadPDS = props.getFileProperty('assembler_loadPDS', buildFile) - impactUtils.saveStaticLinkDependencies(buildFile, assembler_loadPDS, logicalFile) - } - } + // only scan the load module if load module scanning turned on for file + if(!props.userBuild){ + String scanLoadModule = props.getFileProperty('assembler_scanLoadModule', buildFile) + if (scanLoadModule && scanLoadModule.toBoolean()) { + String assembler_loadPDS = props.getFileProperty('assembler_loadPDS', buildFile) + impactUtils.saveStaticLinkDependencies(buildFile, assembler_loadPDS, logicalFile) } } - } - } + // clean up passed DD statements job.stop() @@ -233,19 +266,28 @@ def createAssemblerCICSTranslatorCommand(String buildFile, LogicalFile logicalFi } /* - * createCompileCommand - creates a MVSExec command for compiling the BMS Map (buildFile) + * createCompileCommand - creates a MVSExec command for compiling the source code */ def createAssemblerCommand(String buildFile, LogicalFile logicalFile, String member, File logFile) { + def errPrefixOptions = props.getFileProperty('assembler_compileErrorPrefixParms', buildFile) ?: "" - + def debugOptions = props.getFileProperty('assembler_debugParms', buildFile) ?: "" + String parameters = props.getFileProperty('assembler_pgmParms', buildFile) if (props.errPrefix) parameters = "$parameters,$errPrefixOptions" + if (props.debug) + parameters = "$parameters,$debugOptions" + // define the MVSExec command to compile the BMS map MVSExec assembler = new MVSExec().file(buildFile).pgm(props.assembler_pgm).parm(parameters) + // asma options file + def asmaOpts = props.getFileProperty('assembler_asmaOptFile', buildFile) ?: "" + if (asmaOpts) assembler.dd(new DDStatement().name("ASMAOPT").dsn("${asmaOpts}").options("shr")) + // add DD statements to the compile command String assembler_srcPDS = props.getFileProperty('assembler_srcPDS', buildFile) @@ -296,9 +338,13 @@ def createAssemblerCommand(String buildFile, LogicalFile logicalFile, String mem if (props.SDFSMAC) assembler.dd(new DDStatement().dsn(props.SDFSMAC).options("shr")) + // SYSADATA allocation + if (props.errPrefix || props.debug) { + assembler.dd(new DDStatement().name("SYSADATA").dsn("${props.assembler_sysadataPDS}($member)").options("shr")) + } + // add IDz User Build Error Feedback DDs if (props.errPrefix) { - assembler.dd(new DDStatement().name("SYSADATA").options("DUMMY")) // SYSXMLSD.XML suffix is mandatory for IDZ/ZOD to populate remote error list assembler.dd(new DDStatement().name("SYSXMLSD").dsn("${props.hlq}.${props.errPrefix}.SYSXMLSD.XML").options(props.assembler_compileErrorFeedbackXmlOptions)) } @@ -309,6 +355,22 @@ def createAssemblerCommand(String buildFile, LogicalFile logicalFile, String mem } +/* + * createDebugSideFileCommand - creates a MVSExec command creating a IDILANGX side file + * https://www.ibm.com/docs/en/developer-for-zos/16.0?topic=program-creating-eqalangx-file-assembler + * + */ +def createDebugSideFile(String buildFile, LogicalFile logicalFile, String member, File logFile) { + + String parameters = props.getFileProperty('assembler_eqalangxParms', buildFile) + + MVSExec generateSidefile = new MVSExec().file(buildFile).pgm(props.assembler_eqalangx).parm(parameters) + generateSidefile.dd(new DDStatement().name("TASKLIB").dsn("${props.PDTCCMOD}").options("shr")) + generateSidefile.dd(new DDStatement().name("SYSADATA").dsn("${props.assembler_sysadataPDS}($member)").options("shr")) + generateSidefile.dd(new DDStatement().name("IDILANGX").dsn("${props.assembler_debugPDS}($member)").options("shr").output(true).deployType("EQALANGX")) + return generateSidefile +} + /* * createLinkEditCommand - creates a MVSExec xommand for link editing the assembler object module produced by the compile */ diff --git a/languages/BMS.groovy b/languages/BMS.groovy index f8482f1a..25c0812e 100644 --- a/languages/BMS.groovy +++ b/languages/BMS.groovy @@ -9,7 +9,7 @@ import groovy.transform.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.bms_requiredBuildProperties) @@ -18,11 +18,11 @@ def langQualifier = "bms" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'bms_fileBuildRank') - +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'bms_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // copy build file to input data set buildUtils.copySourceFiles(buildFile, props.bms_srcPDS, null, null, null) diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 5d7cea99..7639304e 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -14,7 +14,7 @@ import com.ibm.dbb.build.report.records.* @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.cobol_requiredBuildProperties) @@ -24,7 +24,8 @@ def langQualifier = "cobol" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'cobol_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'cobol_fileBuildRank') +int currentBuildFileNumber = 1 if (buildListContainsTests(sortedList)) { langQualifier = "cobol_test" @@ -33,7 +34,7 @@ if (buildListContainsTests(sortedList)) { // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // Check if this a testcase isZUnitTestCase = (props.getFileProperty('cobol_testcase', buildFile).equals('true')) ? true : false @@ -52,6 +53,9 @@ sortedList.each { buildFile -> // Get logical file LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) + // print logicalFile details and overrides + if (props.verbose) buildUtils.printLogicalFileAttributes(logicalFile) + // create mvs commands String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.cobol.log") @@ -103,7 +107,7 @@ sortedList.each { buildFile -> // only scan the load module if load module scanning turned on for file String scanLoadModule = props.getFileProperty('cobol_scanLoadModule', buildFile) if (scanLoadModule && scanLoadModule.toBoolean()) - impactUtils.saveStaticLinkDependencies(buildFile, props.linkedit_loadPDS, logicalFile) + impactUtils.saveStaticLinkDependencies(buildFile, props.cobol_loadPDS, logicalFile) } } } diff --git a/languages/DBDgen.groovy b/languages/DBDgen.groovy index 09f38cc8..cfdb69d7 100644 --- a/languages/DBDgen.groovy +++ b/languages/DBDgen.groovy @@ -8,7 +8,7 @@ import groovy.transform.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.dbdgen_requiredBuildProperties) @@ -17,11 +17,12 @@ def langQualifier = "dbdgen" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'dbdgen_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'dbdgen_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // copy build file to input data set buildUtils.copySourceFiles(buildFile, props.dbdgen_srcPDS, null, null, null) diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index 25a04301..8b7c4b06 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -10,7 +10,7 @@ import groovy.transform.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.linkedit_requiredBuildProperties) @@ -19,11 +19,12 @@ def langQualifier = "linkedit" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'linkedit_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'linkedit_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // copy build file to input data set buildUtils.copySourceFiles(buildFile, props.linkedit_srcPDS, null, null, null) diff --git a/languages/MFS.groovy b/languages/MFS.groovy index 86fa4028..c03f1b83 100644 --- a/languages/MFS.groovy +++ b/languages/MFS.groovy @@ -10,7 +10,7 @@ import groovy.transform.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.mfs_requiredBuildProperties) @@ -19,11 +19,12 @@ def langQualifier = "mfs" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'mfs_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'mfs_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // copy build file to input data set buildUtils.copySourceFiles(buildFile, props.mfs_srcPDS, null, null, null) diff --git a/languages/PLI.groovy b/languages/PLI.groovy index 96dfc0ab..e6f924cd 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -13,7 +13,7 @@ import com.ibm.dbb.build.report.records.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.pli_requiredBuildProperties) @@ -22,7 +22,8 @@ def langQualifier = "pli" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'pli_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'pli_fileBuildRank') +int currentBuildFileNumber = 1 if (buildListContainsTests(sortedList)) { langQualifier = "pli_test" @@ -31,7 +32,7 @@ if (buildListContainsTests(sortedList)) { // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // Check if this a testcase isZUnitTestCase = (props.getFileProperty('pli_testcase', buildFile).equals('true')) ? true : false @@ -51,6 +52,9 @@ sortedList.each { buildFile -> // Get logical file LogicalFile logicalFile = buildUtils.createLogicalFile(dependencyResolver, buildFile) + // print logicalFile details and overrides + if (props.verbose) buildUtils.printLogicalFileAttributes(logicalFile) + // create mvs commands String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.pli.log") @@ -99,7 +103,7 @@ sortedList.each { buildFile -> if(!props.userBuild && !isZUnitTestCase){ String scanLoadModule = props.getFileProperty('pli_scanLoadModule', buildFile) if (scanLoadModule && scanLoadModule.toBoolean()) - impactUtils.saveStaticLinkDependencies(buildFile, props.linkedit_loadPDS, logicalFile) + impactUtils.saveStaticLinkDependencies(buildFile, props.pli_loadPDS, logicalFile) } } } diff --git a/languages/PSBgen.groovy b/languages/PSBgen.groovy index 3a7ace11..9f10fa3d 100644 --- a/languages/PSBgen.groovy +++ b/languages/PSBgen.groovy @@ -8,7 +8,7 @@ import groovy.transform.* @Field BuildProperties props = BuildProperties.getInstance() @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.psbgen_requiredBuildProperties) @@ -20,11 +20,12 @@ def acbgenLangQualifier = "acbgen" buildUtils.createLanguageDatasets(acbgenLangQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'psbgen_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'psbgen_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // copy build file to input data set buildUtils.copySourceFiles(buildFile, props.psbgen_srcPDS, null, null, null) diff --git a/languages/REXX.groovy b/languages/REXX.groovy index 55f0b2ec..ad4bd1df 100644 --- a/languages/REXX.groovy +++ b/languages/REXX.groovy @@ -11,7 +11,7 @@ import groovy.transform.* @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.rexx_requiredBuildProperties) @@ -21,11 +21,12 @@ def langQualifier = "rexx" buildUtils.createLanguageDatasets(langQualifier) // sort the build list based on build file rank if provided -List sortedList = buildUtils.sortBuildList(argMap.buildList, 'rexx_fileBuildRank') +List sortedList = buildUtils.sortBuildList(argMap.buildList.sort(), 'rexx_fileBuildRank') +int currentBuildFileNumber = 1 // iterate through build list sortedList.each { buildFile -> - println "*** Building file $buildFile" + println "*** (${currentBuildFileNumber++}/${sortedList.size()}) Building file $buildFile" // configure dependency resolution and create logical file @@ -85,7 +86,7 @@ sortedList.each { buildFile -> // only scan the load module if load module scanning turned on for file String scanLoadModule = props.getFileProperty('rexx_scanLoadModule', buildFile) if (scanLoadModule && scanLoadModule.toBoolean()) - impactUtils.saveStaticLinkDependencies(buildFile, props.linkedit_loadPDS, logicalFile) + impactUtils.saveStaticLinkDependencies(buildFile, props.rexx_loadPDS, logicalFile) } } } diff --git a/languages/Transfer.groovy b/languages/Transfer.groovy index 84ed5ca4..b9ad7f64 100644 --- a/languages/Transfer.groovy +++ b/languages/Transfer.groovy @@ -21,6 +21,17 @@ import groovy.transform.* * * File names cannot exeed more than 8 characters, so they can be stored in * the target dataset. * + * * Review configurations in + * + * build-conf/Transfer.properties + * to define target datasets and dataset characteristics + * + * application-conf/file.properties + * to map files and define the deployType + * + * application-conf/Transfer.properties + * to specify the default deployType + * */ // define script properties @@ -29,16 +40,16 @@ import groovy.transform.* // Set to keep information about which datasets where already checked/created @Field HashSet verifiedBuildDatasets = new HashSet() -println("** Building files mapped to ${this.class.getName()}.groovy script") - +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.transfer_requiredBuildProperties) -List buildList = argMap.buildList +List buildList = argMap.buildList.sort() +int currentBuildFileNumber = 1 // iterate through build list buildList.each { buildFile -> - println "*** Transferring file $buildFile" + println "*** (${currentBuildFileNumber++}/${buildList.size()}) Transferring file $buildFile" // local variables and log file String member = CopyToPDS.createMemberName(buildFile) @@ -55,16 +66,28 @@ buildList.each { buildFile -> // evaluate the datasetmapping, which maps build files to targetDataset defintions PropertyMappings dsMapping = new PropertyMappings("transfer_datasetMapping") + PropertyMappings dsOptionsMapping = new PropertyMappings("transfer_dsOptions") // obtain the target dataset based on the mapped dataset key - String targetDataset = props.getProperty(dsMapping.getValue(buildFile)) - + mappedDatesetDef = dsMapping.getValue(buildFile) + String targetDataset = props.getProperty(mappedDatesetDef) + if (targetDataset != null) { - + + // obtain the dataset reference for targetDataset + String datasetOptions = dsOptionsMapping.getValue(mappedDatesetDef) + + if (datasetOptions == null) { + String errorMsg = "*! Dataset options for $buildFile could not be obtained PropertyMappings . " + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + } + // allocate target dataset if (!verifiedBuildDatasets.contains(targetDataset)) { // using a cache not to allocate all defined datasets verifiedBuildDatasets.add(targetDataset) - buildUtils.createDatasets(targetDataset.split(), props.transfer_srcOptions) + buildUtils.createDatasets(targetDataset.split(), datasetOptions) } // copy the file to the target dataset @@ -72,7 +95,7 @@ buildList.each { buildFile -> try { int rc = new CopyToPDS().file(new File(buildUtils.getAbsolutePath(buildFile))).dataset(targetDataset).member(member).output(true).deployType(deployType).execute() - if (props.verbose) println "** Copyied $buildFile to $targetDataset with deployTyoe $deployType; rc = $rc" + if (props.verbose) println "** Copied $buildFile to $targetDataset with deployType $deployType (rc = $rc)" if (rc!=0){ String errorMsg = "*! The CopyToPDS return code ($rc) for $buildFile exceeded the maximum return code allowed (0)." diff --git a/languages/ZunitConfig.groovy b/languages/ZunitConfig.groovy index 7dcd2e5f..84de7a95 100644 --- a/languages/ZunitConfig.groovy +++ b/languages/ZunitConfig.groovy @@ -11,7 +11,7 @@ import groovy.xml.* @Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) -println("** Building files mapped to ${this.class.getName()}.groovy script") +println("** Building ${argMap.buildList.size()} ${argMap.buildList.size() == 1 ? 'file' : 'files'} mapped to ${this.class.getName()}.groovy script") // verify required build properties buildUtils.assertBuildProperties(props.cobol_requiredBuildProperties) @@ -19,11 +19,11 @@ buildUtils.assertBuildProperties(props.zunit_requiredBuildProperties) def langQualifier = "zunit" buildUtils.createLanguageDatasets(langQualifier) - +int currentBuildFileNumber = 1 // iterate through build list -(argMap.buildList).each { buildFile -> - println "*** Building file $buildFile" +(argMap.buildList.sort()).each { buildFile -> + println "*** (${currentBuildFileNumber++}/${argMap.buildList.size()}) Building file $buildFile" String member = CopyToPDS.createMemberName(buildFile) diff --git a/samples/application-conf/Assembler.properties b/samples/application-conf/Assembler.properties index 7f8fde85..1d0731a7 100644 --- a/samples/application-conf/Assembler.properties +++ b/samples/application-conf/Assembler.properties @@ -10,10 +10,17 @@ assembler_fileBuildRank= # can be overridden by file properties assembler_pgmParms=LIST assembler_linkEditParms=MAP,RENT,COMPAT(PM5) +assembler_debugParms=ADATA assembler_compileErrorPrefixParms=ADATA,EX(ADX(ELAXHASM)) +assembler_eqalangxParms=(ASM ERROR LOUD assembler_db2precompilerParms=HOST(ASM) assembler_cicsprecompilerParms= +# +# (optional) ASMAOPT dataset defining the assembler option list +# can be overridden by file properties +assembler_asmaOptFile= + # Optional linkEditStream defining additional link instructions via SYSIN dd # cobol_linkEditStream= INCLUDE SYSLIB(COBJT) \n assembler_linkEditStream= @@ -24,10 +31,13 @@ assembler_linkEditStream= assembler_linkEdit=true # -# default Assembler maximum RCs allowed +# default Assembler maximum allowed return codes for each step # can be overridden by file properties +assembler_maxSQLTranslatorRC=4 +assembler_maxCICSTranslatorRC=4 assembler_maxRC=4 -assembler_linkEditMaxRC=0 +assembler_maxIDILANGX_RC=8 +assembler_linkEditMaxRC=4 # # lists of properties which should cause a rebuild after being changed diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index 5232fcfb..1a4be3d3 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -67,11 +67,19 @@ Property | Description | Overridable assembler_fileBuildRank | Default Assemble program build rank. Used to sort Assembler build file sub-list. Leave empty. | true assembler_pgmParms | Default Assembler parameters. | true assembler_linkEditParms | Default parameters for the link edit step. | true +assembler_debugParms | Assembler options when the debug flag is set. | true assembler_compileErrorPrefixParms | Default parameters to support remote error feedback in user build scenarios | true +assembler_eqalangxParms | Default parameters for eqalangx utility to produce debug sidefile. | true +assembler_db2precompilerParms | Default Assembler parameters for Db2 precompiler step. | true +assembler_cicsprecompilerParms | Default Assembler parameters for CICS precompiler step. | true +assembler_asmaOptFile | Optional ASMAOPT file - dataset(member). | true assembler_linkEdit | Flag indicating to execute the link edit step to produce a load module for the source file. If false then a object deck will be created instead for later linking. | true assembler_linkEditStream | Optional linkEditStream defining additional link instructions via SYSIN dd | true -assembler_maxRC | Default Assembler maximum RC allowed. | true -assembler_linkEditMaxRC | Default link edit maximum RC allowed. | true +assembler_maxSQLTranslatorRC | Default maximum return code for the sql translator step. | true +assembler_maxCICSTranslatorRC | Default maximum return code for the cics translator step. | true +assembler_maxRC | Default maximum return code for the Assembler step. | true +assembler_maxIDILANGX_RC | Default maximum return code for the debug IDILANGX sidefile generation step. | true +assembler_linkEditMaxRC | Default maximum return code for the linkEdit step. | true assembler_impactPropertyList | List of build properties causing programs to rebuild when changed | false assembler_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false assembler_impactPropertyListSQL | List of SQL build properties causing programs to rebuild when changed | false diff --git a/samples/application-conf/file.properties b/samples/application-conf/file.properties index 7553c3e9..57018c23 100644 --- a/samples/application-conf/file.properties +++ b/samples/application-conf/file.properties @@ -36,6 +36,10 @@ dbb.scannerMapping = ZUnitConfigScanner :: **/*.bzucfg # mapping for overwriting the impactResolution rules in application.properties # impactResolutionRules=[${copybookRule},${linkRule}] :: **/copy/*.cpy,**/cobol/*.cbl +######### +# Configuration for Transfer.groovy which copies files to target datasets and report them +# in the BuildReport +######### # # PropertyMapping to map files using the Transfer.groovy language script to different target datasets # diff --git a/test/applications/HelloWorld/application-conf/Assembler.properties b/test/applications/HelloWorld/application-conf/Assembler.properties new file mode 100644 index 00000000..1d0731a7 --- /dev/null +++ b/test/applications/HelloWorld/application-conf/Assembler.properties @@ -0,0 +1,82 @@ +# Application properties used by zAppBuild/language/Assembler.groovy + +# +# default Assemble program build rank - used to sort language build file list +# leave empty - overridden by file properties if sorting needed +assembler_fileBuildRank= + +# +# default Assembler parameters +# can be overridden by file properties +assembler_pgmParms=LIST +assembler_linkEditParms=MAP,RENT,COMPAT(PM5) +assembler_debugParms=ADATA +assembler_compileErrorPrefixParms=ADATA,EX(ADX(ELAXHASM)) +assembler_eqalangxParms=(ASM ERROR LOUD +assembler_db2precompilerParms=HOST(ASM) +assembler_cicsprecompilerParms= + +# +# (optional) ASMAOPT dataset defining the assembler option list +# can be overridden by file properties +assembler_asmaOptFile= + +# Optional linkEditStream defining additional link instructions via SYSIN dd +# cobol_linkEditStream= INCLUDE SYSLIB(COBJT) \n +assembler_linkEditStream= + +# +# execute link edit step +# can be overridden by file properties +assembler_linkEdit=true + +# +# default Assembler maximum allowed return codes for each step +# can be overridden by file properties +assembler_maxSQLTranslatorRC=4 +assembler_maxCICSTranslatorRC=4 +assembler_maxRC=4 +assembler_maxIDILANGX_RC=8 +assembler_linkEditMaxRC=4 + +# +# lists of properties which should cause a rebuild after being changed +assembler_impactPropertyList=assembler_pgmParms +assembler_impactPropertyListCICS=assembler_db2precompilerParms +assembler_impactPropertyListSQL=assembler_cicsprecompilerParms + +# +# ASM dependencySearch configuration +# searchPaths defined in app-properties file +assembler_dependencySearch=${asmMacroSearch}${asmCopySearch} + +# +# store abbrev git hash in ssi field +# available for buildTypes impactBuild, mergeBuild and fullBuild +# can be overridden by file properties +assembler_storeSSI=true + +# +# default deployType +assembler_deployType=LOAD + +# +# deployType for build files with isCICS=true +assembler_deployTypeCICS=CICSLOAD + +# +# deployType for build files with isDLI=true +assembler_deployTypeDLI=IMSLOAD + +# +# scan link edit load module for link dependencies +# can be overridden by file properties +assembler_scanLoadModule=true + +# +# additional libraries for assembler SYSLIB concatenation, comma-separated +assembler_assemblySyslibConcatenation= + +# +# additional libraries for linkEdit SYSLIB concatenation, comma-separated +assembler_linkEditSyslibConcatenation= diff --git a/test/applications/HelloWorld/application-conf/application.properties b/test/applications/HelloWorld/application-conf/application.properties new file mode 100644 index 00000000..d20f7163 --- /dev/null +++ b/test/applications/HelloWorld/application-conf/application.properties @@ -0,0 +1,201 @@ +# Build properties used by zAppBuild/build.groovy + +# +# Run zUnit Tests +# Defaults to "false", to enable, set to "true" +#runzTests=true + +# +# Comma separated list of additional application property files to load +# Supports both relative path (to ${application}/application-conf/) and absolute path +applicationPropFiles=file.properties,Assembler.properties + +# +# Comma separated list all source directories included in application build. Supports both absolute +# and relative paths. Relative assumed to be relative to ${workspace}. +# ex: applicationSrcDirs=${application},/u/build/common/copybooks +applicationSrcDirs=${application} + +# +# Comma separated list of the build script processing order +buildOrder=BMS.groovy,MFS.groovy,Cobol.groovy,Assembler.groovy,PLI.groovy,LinkEdit.groovy,DBDgen.groovy,PSBgen.groovy,Transfer.groovy + +# +# Comma seperated list of the test script processing order +testOrder=ZunitConfig.groovy + +# +# Flag to log output in table views instead of printing raw JSON data +# See also build-conf/build.properties +# default = false +# formatConsoleOutput=false + +# +# The main build branch. Used for cloning collections for topic branch builds instead +# of rescanning the entire application. +mainBuildBranch=master + +# +# The git repository URL of the application repository to establish links to the changed files +# in the build result properties +# ex: for GitHub: https://github.com/ibm/dbb-zappbuild/ +gitRepositoryURL= + +# +# exclude list used when scanning or running full build +excludeFileList=.*,**/.*,**/*.properties,**/*.xml,**/*.groovy,**/*.json,**/*.md,**/application-conf/*.* + +# +# comma-separated list of file patterns for which impact calculation should be skipped. Uses glob file patterns +# sample: skipImpactCalculationList=**/epsmtout.cpy,**/centralCopybooks/*.cpy +skipImpactCalculationList= + +# +# Job card, please use \n to indicate a line break and use \ to break the line in this property file +# Example: jobCard=//RUNZUNIT JOB ,MSGCLASS=H,CLASS=A,NOTIFY=&SYSUID,REGION=0M +jobCard= + +############################################################### +# Build Property management +############################################################### +# zAppBuild allows you to manage default properties and file properties: +# - Documentation on how to override corresponding default build properties can be found at: +# https://github.com/IBM/dbb-zappbuild/docs/FilePropertyManagement.md + +# ### Properties to enable and configure build property overrides using individual artifact properties files + +# flag to enable the zAppBuild capability to load individual artifact properties files for all individual source files. +# Note: To only activate loadFileLevelProperties for a group of files, it is recommended to use DBB's file property path +# syntax in application-conf/file.properties instead. +# default: false +loadFileLevelProperties=false + +# Property to enable/disable and configure build property overrides using language configuration mapping +# file - languageConfigurationMapping.properties +# If loadFileLevelProperties is set as true above, the properties from the individual artifact properties files will override the +# properties from language configuration properties file. +# Note: To only activate loadLanguageConfigurationProperties for a group of files, it is recommended to use DBB's file property path +# syntax in application-conf/file.properties instead. +loadLanguageConfigurationProperties=false + +# relative path to folder containing individual artifact properties files +# assumed to be relative to ${workspace}/${application} +propertyFilePath=properties + +# file extension for individual artifact properties files +# default: properties +propertyFileExtension=properties + +############################################################### +# Dependency Analysis and Impact Analysis configuration +############################################################### + +# +# boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact +# the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile +# +# default:false +resolveSubsystems=false + +# +# SearchPathImpactFinder resolution searchPath configuration +# list of multiple search path configurations which are defined below +# +# this configuration is used when running zAppBuild with the --impactBuild option +# to calculate impacted files based on the identified changed files +impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${zunitTestConfigSearch}${zunitTestcasePgmSearch} + +# +# copybookSearch +# searchPath to locate Cobol copybooks +# used in dependency resolution and impact analysis +# +# Please be as specific as possible when configuring the searchPath. +# Alternate configurations: +# +# dependency resolution from multiple repositories / multiple root folders: +# copybookSearch = search:${workspace}/?path=**/copybook/*.cpy +# +# dependency resolution across all directories in build workspace, but filtering on the file extension cpy: +# copybookSearch = search:${workspace}/?path=**/*.cpy +# dependency resolution across all directories in build workspace, but filtering on the file extension cpy and cobcpy: +# copybookSearch = search:${workspace}/?path=**/*.cpy;**/*.cobcpy +# +# dependency resolution in the application directory and a shared common copybook location: +# copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy;/u/build/common/copybooks/*.cpy +# +# More samples can be found along with the syntax for the search path configurations at: +# https://www.ibm.com/docs/en/dbb/2.0.0?topic=apis-dependency-impact-resolution#6-resolving-logical-build-dependencies-to-local-physical-files +# +copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy + +# +# pliincludeSearch +# searchPath to locate PLI include files +# used in dependency resolution and impact analysis +pliincludeSearch = search:${workspace}/?path=${application}/plinc/*.cpy + +# asmMacroSearch +# searchPath to locate Assembler macro files +# use category filters on what you want to include during the scan (i.e. excludes macro-def keyword) +# used in dependency resolution and impact analysis +asmMacroSearch = search:[SYSLIB:MACRO]${workspace}/?path=${application}/maclib/*.mac + +# asmCopySearch +# searchPath to locate Assembler copy files +# used in dependency resolution and impact analysis +asmCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/maclib/*.mac + +# +# bmsSearch +# searchPath to locate impacted bms maps +# used only in impact analysis +bmsSearch = search:${workspace}/?path=${application}/bms/*.bms + +# +# rexxCopySearch +# searchPath to locate rexx copy - defaults to the local rexx folder in the main application folder +# used in dependency resolution and impact analysis +rexxCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/rexx/*.rexx + +# +# linkSearch +# +# searchPath to locate impacted linkcards or main programs after an included submodule is changed +# leverages the output collection, which has the dependency info from the executable +# category LINK only; used only in impact analysis +# +# Additional samples: +# +# impact resolution across all directories in build workspace, but filtering on the file extension cbl: +# staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl +# +# impact resolution across all directories in build workspace, but filtering on the file extension cbl and pli (for cobol and pli submodules): +# staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl,**/*.pli +# +# More samples can be found along with the syntax for the search path configurations at: +# https://www.ibm.com/docs/en/dbb/2.0.0?topic=apis-dependency-impact-resolution#6-resolving-logical-build-dependencies-to-local-physical-files +# +# Special case with Dependency Scanner Transfer Control Statement capturing turned on (default is off) +# the scanners detect a static call to the literal, which would need to turn into a new rule for CALL: +# staticCallSearch = search:[:CALL]${workspace}/?path=${application}/cobol/*.cbl +# +linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl + +# zunitTestConfigSearch +# searchPath to locate zunit config files +# used in impact analysis +zunitTestConfigSearch = search:[:ZUNITINC]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli;${application}/testcase/*.cbl;${application}/testcase/*.pli + +# +# zunitPlayfileSearch +# searchPath to locate zunit playback files +# used in dependency resolution +zunitPlayfileSearch = search:[SYSPLAY:]${workspace}/?path=${application}/testplayfiles/*.bzuplay + +# +# zunitTestcasePgmSearch +# searchPath to locate impacted test case programs +# see also build-conf/build.properties -> createTestcaseDependency +# used in impact analysis +zunitTestcasePgmSearch = search:[SYSPROG:PROGRAMDEPENDENCY]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli diff --git a/test/applications/HelloWorld/application-conf/file.properties b/test/applications/HelloWorld/application-conf/file.properties new file mode 100644 index 00000000..7553c3e9 --- /dev/null +++ b/test/applications/HelloWorld/application-conf/file.properties @@ -0,0 +1,48 @@ +# Application script mappings and file property overrides + +# +# Script mappings for all application programs +dbb.scriptMapping = Assembler.groovy :: **/*.asm +dbb.scriptMapping = BMS.groovy :: **/*.bms +dbb.scriptMapping = MFS.groovy :: **/*.mfs +dbb.scriptMapping = PSBgen.groovy :: **/psb/*.asm +dbb.scriptMapping = DBDgen.groovy :: **/dbd/*.asm +dbb.scriptMapping = Cobol.groovy :: **/*.cbl +dbb.scriptMapping = LinkEdit.groovy :: **/*.lnk +dbb.scriptMapping = PLI.groovy :: **/*.pli +dbb.scriptMapping = ZunitConfig.groovy :: **/*.bzucfg +dbb.scriptMapping = Transfer.groovy :: **/*.jcl, **/*.xml + +# +# Scanner mappings for application programs that require a custom scanner +dbb.scannerMapping = ZUnitConfigScanner :: **/*.bzucfg + +# +# General file level overwrites through DBB Build Properties +# isCICS = true :: **/cobol/member.cbl +# isSQL = true :: **/cobol/member.cbl +# isMQ = true :: **/cobol/member.cbl + +# +# Please check for available file property overwrites within samples/application-conf/README.md + +# +# file mapping for generated zUnit Test case programs (Cobol) to use a seperate set of libraries +# cobol_testcase = true :: **/testcase/*.cbl + +# file mapping for generated zUnit Test case programs (PL/I) to use a seperate set of libraries +# pli_testcase = true :: **/testcase/*.pli + +# mapping for overwriting the impactResolution rules in application.properties +# impactResolutionRules=[${copybookRule},${linkRule}] :: **/copy/*.cpy,**/cobol/*.cbl + +# +# PropertyMapping to map files using the Transfer.groovy language script to different target datasets +# +# transfer_datasetMapping = transfer_jclPDS :: **/*.jcl +# transfer_datasetMapping = transfer_xmlPDS :: **/xml/*.* +# +# file mapping for overwriting the default deployType of the Transfer.groovy language script +# +# transfer_deployType = JCL :: **/*.jcl +# transfer_deployType = XML :: **/xml/*.* \ No newline at end of file diff --git a/test/applications/HelloWorld/assembler/hello.asm b/test/applications/HelloWorld/assembler/hello.asm new file mode 100644 index 00000000..8e672bc1 --- /dev/null +++ b/test/applications/HelloWorld/assembler/hello.asm @@ -0,0 +1,9 @@ +HELLO CSECT + USING *,15 + STM 14,12,12(13) + LR 12,15 + USING HELLO,12 + WTO 'HI' + XR 15,15 + RETURN (14,12) + END diff --git a/test/applications/HelloWorld/test.properties b/test/applications/HelloWorld/test.properties new file mode 100644 index 00000000..eb08967e --- /dev/null +++ b/test/applications/HelloWorld/test.properties @@ -0,0 +1,43 @@ +# Test properties for the HelloWorld application + +######################## +# test.groovy properties +######################## +# +# list of test scripts to run for this application +test_testOrder=resetBuild.groovy,\ +fullBuild.groovy,\ +fullBuild_debug.groovy,\ +resetBuild.groovy + +# +# setting the workspace location for the HelloWorld application +workspace =${zAppBuildDir}/test/applications + + +############################# +# fullBuild.groovy properties +############################# +# +# list of programs should be built for a full build for this application +fullBuild_expectedFilesBuilt = hello.asm +# +# list of source datasets (LLQ) that should be deleted during fullBuild.groovy cleanUp +fullBuild_datasetsToCleanUp = ASM,MACRO,DBRM,OBJ,LOAD + + +############################# +# fullBuild_debug.groovy properties +############################# +# +# list of programs should be built for a full build for this application +fullBuild_debug_expectedFilesBuilt = hello.asm +# +# list of source datasets (LLQ) that should be deleted during fullBuild.groovy cleanUp +fullBuild_debug_datasetsToCleanUp = ASM,MACRO,DBRM,OBJ,LOAD,SYSADATA,EQALANGX +# +# valued for testing expected documented build outputs in build report +# the implementation uses the DBB PropertyMappings +# format: value can be a comma separated list of +# comprising :,: : buildfile +fullBuild_debug_expectedBuildOutputs = HELLO:EQALANGX :: hello.asm \ No newline at end of file diff --git a/test/applications/MortgageApplication/test.properties b/test/applications/MortgageApplication/test.properties index 759e7d90..a21531a2 100644 --- a/test/applications/MortgageApplication/test.properties +++ b/test/applications/MortgageApplication/test.properties @@ -11,6 +11,7 @@ mergeBuild.groovy,\ fullBuild.groovy,\ fullBuild_languageConfigurations.groovy,\ impactBuild.groovy,\ +impactBuild_preview.groovy,\ impactBuild_properties.groovy,\ impactBuild_renaming.groovy,\ impactBuild_deletion.groovy,\ @@ -99,6 +100,20 @@ impactBuild_deletion_deletedOutputs = LOAD(EPSCMORT) :: cobol/epscmort.cbl # list of source datasets (LLQ) that should be deleted during impactBuild.groovy cleanUp impactBuild_deletion_datasetsToCleanUp = BMS,COBOL,LINK,COPY,BMS.COPY,DBRM,LOAD,MFS,OBJ,TFORMAT +############################### +# impactBuild_preview.groovy properties +############################### +# +# list of changed source files to test impact builds +impactBuild_preview_changedFiles = copybook/epsmtout.cpy +# +# list of source datasets (LLQ) that should be deleted during impactBuild.groovy cleanUp +impactBuild_preview_datasetsToCleanUp = BMS,COBOL,LINK +# +# Use file properties to associate expected files built to changed files +impactBuild_preview_expectedFilesBuilt = epsmlist.cbl,epscsmrt.cbl,epscmort.cbl,epsmlist.lnk :: copybook/epsmtout.cpy + + ############################# # fullBuild_languageConfigurations.groovy properties # this test script is validating the language configuration capability of dbb-zappbuild diff --git a/test/test.groovy b/test/test.groovy index 02274164..405d5617 100644 --- a/test/test.groovy +++ b/test/test.groovy @@ -102,9 +102,6 @@ def loadBuildProperties(String [] args) { if (options.f) props.propFiles = options.f if (options.o) props.outDir = options.o - // load application test.properties file - props.load(new File("${getScriptDir()}/applications/${props.app}/test.properties")) - // add some additional properties props.testBranch = 'zAppBuildTesting' props.zAppBuildDir = new File(getScriptDir()).getParent() @@ -126,6 +123,10 @@ def loadBuildProperties(String [] args) { props.appLocation = "${props.zAppBuildDir}/samples/${props.app}" as String props.workspace = "${props.zAppBuildDir}/samples" as String + // load application test.properties file + props.load(new File("${getScriptDir()}/applications/${props.app}/test.properties")) + + // print properties if (props.verbose) { println "** Properties args and applications/${props.app}/test.properties" diff --git a/test/testScripts/README.md b/test/testScripts/README.md index 034fac5a..7456b4bd 100644 --- a/test/testScripts/README.md +++ b/test/testScripts/README.md @@ -10,6 +10,12 @@ This script is called by test.groovy to run a full build and multiple user build - userBuild TC1 to have a successful override of a previously defined file level property - userBuild TC2 to validate a failing override and check on an expected warning message +## fullBuild_debug.groovy +This script is called by test.groovy to run a full build by creating a new “automation” branch from the feature branch specified in the command line argument. It verifies the below requirements +- Full build with the `--debug` cli option ran clean +- Number of expected build files equal the number of files build during the full build in the console. +- Checks if output files are documented in the BuildReport.json. Specifically use it to check if the SIDEFILE for Assembler is present. Cobol and PLI don't produce an additional output file. + ## impactBuild.groovy This script that is called by test.groovy to run an impact build against all the program file specified in the [test.properties](/test/applications/MortgageApplication/test.properties). It verifies the below requirements - Impact build ran clean diff --git a/test/testScripts/fullBuild_debug.groovy b/test/testScripts/fullBuild_debug.groovy new file mode 100644 index 00000000..9fa7f5c7 --- /dev/null +++ b/test/testScripts/fullBuild_debug.groovy @@ -0,0 +1,112 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import groovy.transform.* +import com.ibm.dbb.* +import com.ibm.dbb.build.* +import com.ibm.jzos.ZFile + +@Field BuildProperties props = BuildProperties.getInstance() + +@Field def testUtils = loadScript(new File("../utils/testUtilities.groovy")) + +println "\n** Executing test script fullBuild_debug.groovy" + +// Get the DBB_HOME location +def dbbHome = EnvVars.getHome() +if (props.verbose) println "** DBB_HOME = ${dbbHome}" + +// Create full build command +def fullBuildCommand = [] +fullBuildCommand << "${dbbHome}/bin/groovyz" +fullBuildCommand << "${props.zAppBuildDir}/build.groovy" +fullBuildCommand << "--workspace ${props.workspace}" +fullBuildCommand << "--application ${props.app}" +fullBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +fullBuildCommand << "--hlq ${props.hlq}" +fullBuildCommand << "--logEncoding UTF-8" +fullBuildCommand << "--url ${props.url}" +fullBuildCommand << "--id ${props.id}" +fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +fullBuildCommand << "--verbose" +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +fullBuildCommand << "--fullBuild" +fullBuildCommand << "--debug" + +// Run full build +println "** Executing ${fullBuildCommand.join(" ")}" +def process = ['bash', '-c', fullBuildCommand.join(" ")].execute() +def outputStream = new StringBuffer(); +process.waitForProcessOutput(outputStream, System.err) + +//Validate build results +println "** Validating full build results" +def expectedFilesBuiltList = props.fullBuild_debug_expectedFilesBuilt.split(',') + +@Field def assertionList = [] +PropertyMappings expectedBuildOutputsMapping = new PropertyMappings('fullBuild_debug_expectedBuildOutputs') + +try { + // Validate clean build + assert outputStream.contains("Build State : CLEAN") : "*! FULL BUILD WITH DEBUG FAILED\nOUTPUT STREAM:\n$outputStream\n" + + // Validate expected number of files built + def numFullFiles = expectedFilesBuiltList.size() + assert outputStream.contains("Total files processed : ${numFullFiles}") : "*! TOTAL FILES PROCESSED ARE NOT EQUAL TO ${numFullFiles}\nOUTPUT STREAM:\n$outputStream\n" + + // Validate expected built files in output stream + assert expectedFilesBuiltList.count{ i-> outputStream.contains(i) } == expectedFilesBuiltList.size() : "*! FILES PROCESSED IN THE FULL BUILD DOES NOT CONTAIN THE LIST OF FILES PASSED ${expectedFilesBuiltList}\nOUTPUT STREAM:\n$outputStream\n" + + // Validate the expected outputs + def buildReportFile = testUtils.getBuildReportFromStream(outputStream) + if (buildReportFile) { + def buildReport = testUtils.parseBuildReport(buildReportFile) + if (buildReport) { + expectedFilesBuiltList.each{ expectedFile -> + def expectedOutputs = expectedBuildOutputsMapping.getValue(expectedFile) + expectedOutputs.split(",").each { expectedOutput -> + (member, deployType) = expectedOutput.split(":") + assert testUtils.buildReportIncludesOutput(buildReport, member, deployType) : "*! EXPECTED OUTPUT ($member) with deployType ($deployType) not found in buildreport \nOUTPUT STREAM:\n$outputStream\n" + } + } + } + } + + + println "**" + println "** FULL BUILD TEST : PASSED **" + println "**" +} +catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' +} +finally { + cleanUpDatasets() + if (assertionList.size()>0) { + println "\n***" + println "**START OF FAILED FULL BUILD WITH DEBUG TEST RESULTS**\n" + println "*FAILED FULL BUILD WITH DEBUG RESULTS*\n" + assertionList + println "\n**END OF FAILED FULL BUILD WITH DEBUG**" + println "***" + } + +} + +// script end + +//************************************************************* +// Method Definitions +//************************************************************* + +def cleanUpDatasets() { + def segments = props.fullBuild_debug_datasetsToCleanUp.split(',') + + println "Deleting full build PDSEs ${segments}" + segments.each { segment -> + def pds = "'${props.hlq}.${segment}'" + if (ZFile.dsExists(pds)) { + if (props.verbose) println "** Deleting ${pds}" + ZFile.remove("//$pds") + } + } +} diff --git a/test/testScripts/impactBuild_preview.groovy b/test/testScripts/impactBuild_preview.groovy new file mode 100644 index 00000000..a65187f1 --- /dev/null +++ b/test/testScripts/impactBuild_preview.groovy @@ -0,0 +1,173 @@ + +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import groovy.transform.* +import com.ibm.dbb.* +import com.ibm.dbb.build.* +import com.ibm.jzos.ZFile + +@Field BuildProperties props = BuildProperties.getInstance() +println "\n** Executing test script impactBuild_preview.groovy" + +// Get the DBB_HOME location +def dbbHome = EnvVars.getHome() +if (props.verbose) println "** DBB_HOME = ${dbbHome}" + +// Create full build command to set baseline +def fullBuildCommand = [] +fullBuildCommand << "${dbbHome}/bin/groovyz" +fullBuildCommand << "${props.zAppBuildDir}/build.groovy" +fullBuildCommand << "--workspace ${props.workspace}" +fullBuildCommand << "--application ${props.app}" +fullBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +fullBuildCommand << "--hlq ${props.hlq}" +fullBuildCommand << "--logEncoding UTF-8" +fullBuildCommand << "--url ${props.url}" +fullBuildCommand << "--id ${props.id}" +fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +fullBuildCommand << (props.verbose ? "--verbose" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +fullBuildCommand << "--fullBuild" + +// create impact build command for preview +def impactBuildPreviewCommand = [] +impactBuildPreviewCommand << "${dbbHome}/bin/groovyz" +impactBuildPreviewCommand << "${props.zAppBuildDir}/build.groovy" +impactBuildPreviewCommand << "--workspace ${props.workspace}" +impactBuildPreviewCommand << "--application ${props.app}" +impactBuildPreviewCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +impactBuildPreviewCommand << "--hlq ${props.hlq}" +impactBuildPreviewCommand << "--logEncoding UTF-8" +impactBuildPreviewCommand << "--url ${props.url}" +impactBuildPreviewCommand << "--id ${props.id}" +impactBuildPreviewCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +impactBuildPreviewCommand << (props.verbose ? "--verbose" : "") +impactBuildPreviewCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +impactBuildPreviewCommand << "--impactBuild --preview" // this will run zAppBuild only in preview mode. + +// create impact build command +def impactBuildCommand = [] +impactBuildCommand << "${dbbHome}/bin/groovyz" +impactBuildCommand << "${props.zAppBuildDir}/build.groovy" +impactBuildCommand << "--workspace ${props.workspace}" +impactBuildCommand << "--application ${props.app}" +impactBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +impactBuildCommand << "--hlq ${props.hlq}" +impactBuildCommand << "--logEncoding UTF-8" +impactBuildCommand << "--url ${props.url}" +impactBuildCommand << "--id ${props.id}" +impactBuildCommand << (props.pw ? "--pw ${props.pw}" : "--pwFile ${props.pwFile}") +impactBuildCommand << (props.verbose ? "--verbose" : "") +impactBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +impactBuildCommand << "--impactBuild" + +// iterate through change files to test impact build +@Field def assertionList = [] +PropertyMappings filesBuiltMappings = new PropertyMappings('impactBuild_preview_expectedFilesBuilt') +def changedFiles = props.impactBuild_preview_changedFiles.split(',') +println("** Processing changed files from impactBuild_preview_changedFiles property : ${props.impactBuild_preview_changedFiles}") +try { + + println "\n** Running full build to set baseline" + + // run impact build + println "** Executing ${fullBuildCommand.join(" ")}" + def outputStream = new StringBuffer() + def process = [ + 'bash', + '-c', + fullBuildCommand.join(" ") + ].execute() + process.waitForProcessOutput(outputStream, System.err) + + changedFiles.each { changedFile -> + println "\n** Running IMPACT BUILD WITH PREVIEW TEST for changed file $changedFile" + + // update changed file in Git repo test branch + copyAndCommit(changedFile) + + // run impact build with preview + println "** Executing ${impactBuildPreviewCommand.join(" ")}" + outputStream = new StringBuffer() + process = ['bash', '-c', impactBuildPreviewCommand.join(" ")].execute() + process.waitForProcessOutput(outputStream, System.err) + + validateImpactBuild(changedFile, filesBuiltMappings, outputStream) + + // run impact build + println "** Executing ${impactBuildCommand.join(" ")}" + outputStream = new StringBuffer() + process = ['bash', '-c', impactBuildCommand.join(" ")].execute() + process.waitForProcessOutput(outputStream, System.err) + + // validate build results + // still expecting the same files being impacted + validateImpactBuild(changedFile, filesBuiltMappings, outputStream) + } +} +finally { + cleanUpDatasets() + if (assertionList.size()>0) { + println "\n***" + println "**START OF FAILED IMPACT BUILD WITH PREVIEW TEST RESULTS**\n" + println "*FAILED IMPACT BUILD WITH PREVIEW TEST RESULTS*\n" + assertionList + println "\n**END OF FAILED IMPACT BUILD WITH PREVIEW TEST RESULTS**" + println "***" + } +} +// script end + +//************************************************************* +// Method Definitions +//************************************************************* + +def copyAndCommit(String changedFile) { + println "** Updating and committing ${props.appLocation}/${changedFile}" + def commands = """ + echo ' ' >> ${props.appLocation}/${changedFile} + cd ${props.appLocation}/ + git add . + git commit . -m "edited program file" +""" + def task = ['bash', '-c', commands].execute() + def outputStream = new StringBuffer(); + task.waitForProcessOutput(outputStream, System.err) +} + +def validateImpactBuild(String changedFile, PropertyMappings filesBuiltMappings, StringBuffer outputStream) { + + println "** Validating impact build results" + def expectedFilesBuiltList = filesBuiltMappings.getValue(changedFile).split(',') + + try{ + // Validate clean build + assert outputStream.contains("Build State : CLEAN") : "*! IMPACT BUILD FAILED FOR $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + // Validate expected number of files built + def numImpactFiles = expectedFilesBuiltList.size() + assert outputStream.contains("Total files processed : ${numImpactFiles}") : "*! IMPACT BUILD FOR $changedFile TOTAL FILES PROCESSED ARE NOT EQUAL TO ${numImpactFiles}\nOUTPUT STREAM:\n$outputStream\n" + + // Validate expected built files in output stream + assert expectedFilesBuiltList.count{ i-> outputStream.contains(i) } == expectedFilesBuiltList.size() : "*! IMPACT BUILD FOR $changedFile DOES NOT CONTAIN THE LIST OF BUILT FILES EXPECTED ${expectedFilesBuiltList}\nOUTPUT STREAM:\n$outputStream\n" + + println "**" + println "** IMPACT BUILD WITH PREVIEW TEST : PASSED FOR $changedFile **" + println "**" + } + catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' + } +} +def cleanUpDatasets() { + def segments = props.impactBuild_preview_datasetsToCleanUp.split(',') + + println "Deleting impact build PDSEs ${segments}" + segments.each { segment -> + def pds = "'${props.hlq}.${segment}'" + if (ZFile.dsExists(pds)) { + if (props.verbose) println "** Deleting ${pds}" + ZFile.remove("//$pds") + } + } +} diff --git a/test/utils/README.md b/test/utils/README.md new file mode 100644 index 00000000..0323f8de --- /dev/null +++ b/test/utils/README.md @@ -0,0 +1,5 @@ +## Utilities available for test scripts + +File | Description +--- | --- +[testUtilities.groovy](testUtilities.groovy) | Utilities script containing common methods which can be used across build scripts. \ No newline at end of file diff --git a/test/utils/testUtilities.groovy b/test/utils/testUtilities.groovy new file mode 100644 index 00000000..f78fd2cf --- /dev/null +++ b/test/utils/testUtilities.groovy @@ -0,0 +1,86 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import java.io.File +import com.ibm.dbb.build.* +import java.util.regex.Pattern +import java.util.regex.Matcher +import com.ibm.dbb.build.report.BuildReport +import com.ibm.dbb.build.report.records.* + +/* + * testUtilities + * + * - getBuildReportFromStream allows passing the outputstream and retrieve the location of the buildReport + * - parseBuildReport parses the build report from file and returns the buildReport object + * - buildReportIncludesOutput validating if a provided build report contains a provided output file + * + */ + +/** + * + * Extracts the path to the build report from the console log + * + * @param outputStream + * @return buildReportLocation + * if not found + * + */ +def getBuildReportFromStream(StringBuffer outputStream) { + + def pattern = Pattern.compile("Writing build report data to (\\/.*.json)") + // build matcher and find + Matcher matcher = pattern.matcher(outputStream) + + // get build report path + if(matcher.find()) { + buildReportLocation = matcher.group(1) + return buildReportLocation + } + println("!* The location of the Build report could not be found.") + return null +} + +/** + * + * parses the build report from a provided String + * + * @param buildReport + * @return + */ + +def parseBuildReport(String buildReport) { + def buildReportFile = new File(buildReport) + if (buildReportFile.exists()) { + def parsedBuildReport = BuildReport.parse(new FileInputStream(buildReportFile)) + return parsedBuildReport + } + return null +} + +/** + * buildReportIncludesOutput validates if the provided member with deployType exist on the buildReport + * + * @param buildReport + * @param member + * @param deployType + * @return boolean - if member/deployType was found in build report + * + */ +boolean buildReportIncludesOutput(BuildReport buildReport, String member, String deployType) { + + def buildRecords = buildReport.getRecords().findAll{ + it.getType()==DefaultRecordFactory.TYPE_EXECUTE + } + + def matchingRecord = buildRecords.find{ report -> + report.getOutputs().find{ o -> + o.deployType == deployType && + o.dataset.contains(member) + } + } + + if (matchingRecord != null) { + return true + } else { + return false + } +} \ No newline at end of file diff --git a/utilities/BuildReportUtilities.groovy b/utilities/BuildReportUtilities.groovy index d7bece44..1bd4fbe5 100644 --- a/utilities/BuildReportUtilities.groovy +++ b/utilities/BuildReportUtilities.groovy @@ -45,20 +45,36 @@ def processDeletedFilesList(List deletedList){ List deletedOutputsList = new ArrayList() - String outputLibs = props.getFileProperty("${langPrefix}_outputDatasets", deletedFile) - outputLibs.split(',').each{ outputDS -> - // record for deleted dataset(member) - String outputRecord = "$outputDS"+"($member)" - if (props.verbose) println "** Document deletion ${outputRecord} for file ${deletedFile}" - deletedOutputsList.add(outputRecord) + String outputLibs + // obtain output libraries + if (langPrefix == "transfer") { + // obtain the mapped dataset of the target dataset + PropertyMappings dsMapping = new PropertyMappings("transfer_datasetMapping") + def mappedDatesetDef = dsMapping.getValue(deletedFile) + outputLibs = props.getProperty(mappedDatesetDef) + } else { + // the defaul evaluates {langPrefix}_outputDatasets + outputLibs = props.getFileProperty("${langPrefix}_outputDatasets", deletedFile) + } + + if (outputLibs != null) { + outputLibs.split(',').each{ outputDS -> + // record for deleted dataset(member) + String outputRecord = "$outputDS"+"($member)" + if (props.verbose) println "** Document deletion ${outputRecord} for file ${deletedFile}" + deletedOutputsList.add(outputRecord) - // delete outputRecord from build datasets - if (ZFile.dsExists("//'$outputRecord'")) { - if (props.verbose) println "** Deleting ${outputRecord}" - ZFile.remove("//'$outputRecord'") - } + // delete outputRecord from build datasets + if (ZFile.dsExists("//'$outputRecord'")) { + if (props.verbose) println "** Deleting ${outputRecord}" + ZFile.remove("//'$outputRecord'") + } + } + } else { + println "** No output library found for $deletedFile" } + if(deletedOutputsList.size() > 0 ) { deleteRecord.setAttribute("deletedBuildOutputs",deletedOutputsList) diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index b26a5929..6599bcd8 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -9,6 +9,9 @@ import groovy.json.JsonSlurper import com.ibm.dbb.build.DBBConstants.CopyMode import com.ibm.dbb.build.report.records.* import com.ibm.jzos.FileAttribute +import java.nio.file.FileSystems +import java.nio.file.Path +import java.nio.file.PathMatcher import groovy.ant.* // define script properties @@ -26,7 +29,7 @@ def assertBuildProperties(String requiredProps) { buildProps.each { buildProp -> buildProp = buildProp.trim() - assert props."$buildProp" : "*! Missing required build property '$buildProp'" + assert (props."$buildProp" || !(new PropertyMappings("$buildProp").getValues().isEmpty())) : "*! Missing required build property '$buildProp'" } } } @@ -539,6 +542,9 @@ def getLangPrefix(String scriptName){ case "PSBgen.groovy": langPrefix = 'psbgen' break; + case "Transfer.groovy": + langPrefix = 'transfer' + break; default: if (props.verbose) println ("*** ! No language prefix defined for $scriptName.") break; @@ -786,4 +792,67 @@ def getShortGitHash(String buildFile) { if (abbrevGitHash != null ) return abbrevGitHash if (props.verbose) println "*! Could not obtain abbreviated githash for buildFile $buildFile" return null -} \ No newline at end of file +} + +/** + * createPathMatcherPattern + * Generic method to build PathMatcher from a build property + */ + +def createPathMatcherPattern(String property) { + List pathMatchers = new ArrayList() + if (property) { + property.split(',').each{ filePattern -> + if (!filePattern.startsWith('glob:') || !filePattern.startsWith('regex:')) + filePattern = "glob:$filePattern" + PathMatcher matcher = FileSystems.getDefault().getPathMatcher(filePattern) + pathMatchers.add(matcher) + } + } + return pathMatchers +} + +/** + * matches + * Generic method to validate if a file is matching any pathmatchers + * + */ +def matches(String file, List pathMatchers) { + def result = pathMatchers.any { matcher -> + Path path = FileSystems.getDefault().getPath(file); + if ( matcher.matches(path) ) + { + return true + } + } + return result +} + +/** + * method to print the logicalFile attributes (CICS, SQL, DLI, MQ) of a scanned file + * and indicating if an attribute is overridden through a property definition. + * + * sample output: + * Program attributes: CICS=true, SQL=true*, DLI=false, MQ=false + * + * additional notes: + * An suffixed asterisk (*) of the value for an attribute is indicating if a property definition + * is overriding the value. When the values are identical, no asterisk is presented, even when + * a property is setting the same value. + * + * This is implementing + * https://github.com/IBM/dbb-zappbuild/issues/339 + * +*/ + +def printLogicalFileAttributes(LogicalFile logicalFile) { + String cicsFlag = (logicalFile.isCICS() == isCICS(logicalFile)) ? "${logicalFile.isCICS()}" : "${isCICS(logicalFile)}*" + String sqlFlag = (logicalFile.isSQL() == isSQL(logicalFile)) ? "${logicalFile.isSQL()}" : "${isSQL(logicalFile)}*" + String dliFlag = (logicalFile.isDLI() == isDLI(logicalFile)) ? "${logicalFile.isDLI()}" : "${isDLI(logicalFile)}*" + String mqFlag = (logicalFile.isMQ() == isMQ(logicalFile)) ? "${logicalFile.isMQ()}" : "${isMQ(logicalFile)}*" + + println "Program attributes: CICS=$cicsFlag, SQL=$sqlFlag, DLI=$dliFlag, MQ=$mqFlag" + +} + + \ No newline at end of file diff --git a/utilities/ImpactUtilities.groovy b/utilities/ImpactUtilities.groovy index 6ba499ba..b11783fd 100644 --- a/utilities/ImpactUtilities.groovy +++ b/utilities/ImpactUtilities.groovy @@ -72,7 +72,7 @@ def createImpactBuildList() { if (props.verbose) println "** Performing impact analysis on changed file $changedFile" // get exclude list - List excludeMatchers = createPathMatcherPattern(props.excludeFileList) + List excludeMatchers = buildUtils.createPathMatcherPattern(props.excludeFileList) // list of impacts String impactSearch = props.getFileProperty('impactSearch', changedFile) @@ -85,7 +85,7 @@ def createImpactBuildList() { // only add impacted files that have a build script mapped to it if (ScriptMappings.getScriptName(impactFile)) { // only add impacted files, that are in scope of the build. - if (!matches(impactFile, excludeMatchers)){ + if (!buildUtils.matches(impactFile, excludeMatchers)){ // calculate abbreviated gitHash for impactFile filePattern = FileSystems.getDefault().getPath(impactFile).getParent().toString() @@ -129,7 +129,7 @@ def createImpactBuildList() { // get excludeListe - List excludeMatchers = createPathMatcherPattern(props.excludeFileList) + List excludeMatchers = buildUtils.createPathMatcherPattern(props.excludeFileList) logicalFileList.each { logicalFile -> def impactFile = logicalFile.getFile() @@ -137,7 +137,7 @@ def createImpactBuildList() { // only add impacted files that have a build script mapped to it if (ScriptMappings.getScriptName(impactFile)) { // only add impacted files, that are in scope of the build. - if (!matches(impactFile, excludeMatchers)){ + if (!buildUtils.matches(impactFile, excludeMatchers)){ buildSet.add(impactFile) if (props.verbose) println "** $impactFile is impacted by changed property $changedProp. Adding to build list." } @@ -404,13 +404,13 @@ def calculateChangedFiles(BuildResult lastBuildResult, boolean calculateConcurre def mode = null // make sure file is not an excluded file - List excludeMatchers = createPathMatcherPattern(props.excludeFileList) + List excludeMatchers = buildUtils.createPathMatcherPattern(props.excludeFileList) if (props.verbose) println "*** Changed files for directory $dir $msg:" changed.each { file -> (file, mode) = fixGitDiffPath(file, dir, true, null) if ( file != null ) { - if ( !matches(file, excludeMatchers)) { + if ( !buildUtils.matches(file, excludeMatchers)) { changedFiles << file if (!calculateConcurrentChanges) githashBuildableFilesMap.addFilePattern(abbrevCurrent, file) if (props.verbose) println "**** $file" @@ -427,7 +427,7 @@ def calculateChangedFiles(BuildResult lastBuildResult, boolean calculateConcurre if (props.verbose) println "*** Deleted files for directory $dir $msg:" deleted.each { file -> - if ( !matches(file, excludeMatchers)) { + if ( !buildUtils.matches(file, excludeMatchers)) { (file, mode) = fixGitDiffPath(file, dir, false, mode) deletedFiles << file if (props.verbose) println "**** $file" @@ -436,7 +436,7 @@ def calculateChangedFiles(BuildResult lastBuildResult, boolean calculateConcurre if (props.verbose) println "*** Renamed files for directory $dir $msg:" renamed.each { file -> - if ( !matches(file, excludeMatchers)) { + if ( !buildUtils.matches(file, excludeMatchers)) { (file, mode) = fixGitDiffPath(file, dir, false, mode) renamedFiles << file if (props.verbose) println "**** $file" @@ -494,293 +494,8 @@ def scanOnlyStaticDependencies(List buildList){ -/** - * Method to calculate and report the changes between the current configuration and concurrent configurations; - * leverages the existing infrastructure to calculateChangedFiles - in this case for concurrent configs. - * - * Invokes method generateConcurrentChangesReports to produce the reports - * - * @param buildSet - * - */ -def calculateConcurrentChanges(Set buildSet) { - - // initialize patterns - List gitRefMatcherPatterns = createMatcherPatterns(props.reportConcurrentChangesGitBranchReferencePatterns) - - // obtain all current remote branches - // TODO: Handle / Exclude branches from other repositories - Set remoteBranches = new HashSet() - props.applicationSrcDirs.split(",").each { dir -> - dir = buildUtils.getAbsolutePath(dir) - remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) - } - - // Run analysis for each remoteBranch, which matches the configured criteria - remoteBranches.each { gitReference -> - - if (matchesPattern(gitReference,gitRefMatcherPatterns) && !gitReference.equals(props.applicationCurrentBranch)){ - - Set concurrentChangedFiles = new HashSet() - Set concurrentRenamedFiles = new HashSet() - Set concurrentDeletedFiles = new HashSet() - Set concurrentBuildProperties = new HashSet() - - if (props.verbose) println "*** Analysing and validating changes for branch $gitReference ." - - (concurrentChangedFiles, concurrentRenamedFiles, concurrentDeletedFiles, concurrentBuildProperties) = calculateChangedFiles(null, true, gitReference) - - // generate reports and verify for intersects - generateConcurrentChangesReports(buildSet, concurrentChangedFiles, concurrentRenamedFiles, concurrentDeletedFiles, gitReference) - - } - } - - } - -/* - * Method to generate the Concurrent Changes reports and validate if the current build list intersects with concurrent changes - */ - -def generateConcurrentChangesReports(Set buildList, Set concurrentChangedFiles, Set concurrentRenamedFiles, Set concurrentDeletedFiles, String gitReference){ - String concurrentChangesReportLoc = "${props.buildOutDir}/report_concurrentChanges.txt" - - File concurrentChangesReportFile = new File(concurrentChangesReportLoc) - String enc = props.logEncoding ?: 'IBM-1047' - concurrentChangesReportFile.withWriterAppend(enc) { writer -> - - if (!(concurrentChangedFiles.size() == 0 && concurrentRenamedFiles.size() == 0 && concurrentDeletedFiles.size() == 0)) { - - if (props.verbose) println("** Writing report of concurrent changes to $concurrentChangesReportLoc for configuration $gitReference") - - writer.write("\n=============================================== \n") - writer.write("** Report for configuration: $gitReference \n") - writer.write("========\n") - - if (concurrentChangedFiles.size() != 0) { - writer.write("** Changed Files \n") - concurrentChangedFiles.each { file -> - if (props.verbose) println " Changed: ${file}" - if (buildList.contains(file)) { - writer.write("* $file is changed and intersects with the current build list.\n") - String msg = "*!! $file is changed on branch $gitReference and intersects with the current build list." - println msg - - // update build result - if (props.reportConcurrentChangesIntersectionFailsBuild && props.reportConcurrentChangesIntersectionFailsBuild.toBoolean()) { - props.error = "true" - buildUtils.updateBuildResult(errorMsg:msg) - } else { - buildUtils.updateBuildResult(warningMsg:msg) - } - } - else - writer.write(" $file\n") - } - } - - if (concurrentRenamedFiles.size() != 0) { - writer.write("** Renamed Files \n") - concurrentRenamedFiles.each { file -> - if (props.verbose) println " Renamed: ${file}" - if (buildList.contains(file)) { - writer.write("* $file got renamed and intersects with the current build list.\n") - String msg = "*!! $file is renamed on branch $gitReference and intersects with the current build list." - println msg - - // update build result - if (props.reportConcurrentChangesIntersectionFailsBuild && props.reportConcurrentChangesIntersectionFailsBuild.toBoolean()) { - props.error = "true" - buildUtils.updateBuildResult(errorMsg:msg) - } else { - buildUtils.updateBuildResult(warningMsg:msg) - } - } - else - writer.write(" $file\n") - } - } - - if (concurrentDeletedFiles.size() != 0) { - writer.write("** Deleted Files \n") - concurrentDeletedFiles.each { file -> - if (props.verbose) println " Deleted: ${file}" - if (buildList.contains(file)) { - writer.write("* $file is deleted and intersects with the current build list.\n") - String msg = "*!! $file is deleted on branch $gitReference and intersects with the current build list." - println msg - - // update build result - if (props.reportConcurrentChangesIntersectionFailsBuild && props.reportConcurrentChangesIntersectionFailsBuild.toBoolean()) { - props.error = "true" - buildUtils.updateBuildResult(errorMsg:msg) - } else { - buildUtils.updateBuildResult(warningMsg:msg) - } - } - else - writer.write(" $file\n") - } - } - } - } -} - -/** - * Method to query the DBB collections with a list of files - * Configured through reportExternalImpacts* build properties - */ - -def reportExternalImpacts(Set changedFiles){ - // query external collections to produce externalImpactList - - Map collectionImpactsSetMap = new HashMap() // - Set impactedFiles = new HashSet() - - List externalImpactReportingList = new ArrayList() - - if (props.verbose) println("*** Running external impact analysis with file filter ${props.reportExternalImpactsAnalysisFileFilter} and collection patterns ${props.reportExternalImpactsCollectionPatterns} with analysis mode ${props.reportExternalImpactsAnalysisDepths}") - - - try { - - if (props.reportExternalImpactsAnalysisDepths == "simple" || props.reportExternalImpactsAnalysisDepths == "deep"){ - - // get directly impacted candidates first - if (props.verbose) println("*** Running external impact analysis for files ") - - // calculate and collect external impacts - changedFiles.each{ changedFile -> - - List fileMatchers = createPathMatcherPattern(props.reportExternalImpactsAnalysisFileFilter) - - // check that file is on reportExternalImpactsAnalysisFileFilter - if(matches(changedFile, fileMatchers)){ - - // get directly impacted candidates first - if (props.verbose) println(" $changedFile ") - - externalImpactReportingList.add(changedFile) - } - else { - if (props.verbose) println("*** Analysis and reporting has been skipped for changed file $changedFile due to build framework configuration (see configuration of build property reportExternalImpactsAnalysisFileFilter)") - } - } - - if (externalImpactReportingList.size() != 0) { - (collectionImpactsSetMap, impactedFiles) = calculateLogicalImpactedFiles(externalImpactReportingList, changedFiles, collectionImpactsSetMap, "***", "buildSet") - - - // get impacted files of idenfied impacted files - if (props.reportExternalImpactsAnalysisDepths == "deep") { - if (props.verbose) println("**** Running external impact analysis for identified external impacted files as dependent files of the initial set. ") - impactedFiles.each{ impactedFile -> - if (props.verbose) println(" $impactedFile ") - - } - def impactsBin - (collectionImpactsSetMap, impactsBin) = calculateLogicalImpactedFiles(new ArrayList(impactedFiles), changedFiles, collectionImpactsSetMap, "****", "impactSet") - } - - } - - // generate reports by collection / application - collectionImpactsSetMap.each{ entry -> - externalImpactList = entry.value - if (externalImpactList.size()!=0){ - // write impactedFiles per application to build workspace - String impactListFileLoc = "${props.buildOutDir}/externalImpacts_${entry.key}.${props.buildListFileExt}" - if (props.verbose) println("*** Writing report of external impacts to file $impactListFileLoc") - File impactListFile = new File(impactListFileLoc) - String enc = props.logEncoding ?: 'IBM-1047' - impactListFile.withWriter(enc) { writer -> - externalImpactList.each { file -> - // if (props.verbose) println file - writer.write("$file\n") - } - } - } - } - - } - else { - println("*! build property reportExternalImpactsAnalysisDepths has an invalid value : ${props.reportExternalImpactsAnaylsisDepths} , valid: simple | deep") - } - - } catch (Exception e) { - println("*! (ImpactUtilities.reportExternalImpacts) Exception caught during reporting of external impacts. Build continues.") - println(e.getMessage()) - } -} - -/* - * Used to inspect dbb collections for potential impacts, sub-method to reportExternalImpacts - */ - -def calculateLogicalImpactedFiles(List fileList, Set changedFiles, Map collectionImpactsSetMap, String indentationMsg, String analysisMode) { - MetadataStore metadataStore = MetadataStoreFactory.getMetadataStore() - - // local matchers to inspect files and collections - List collectionMatcherPatterns = createMatcherPatterns(props.reportExternalImpactsCollectionPatterns) - - // local - List logicalDependencies = new ArrayList() - - // will be returned - Set impactedFiles = new HashSet() - - // creating a list logical dependencies - fileList.each{ file -> - // go after all the files passed in; assess the identified impacted files to skip analysis for files from an impactSet which are on the changed files - if(analysisMode.equals('buildSet') || (analysisMode.equals('impactSet') && !changedFiles.contains(file))){ - String memberName = CopyToPDS.createMemberName(file) - def ldepFile = new LogicalDependency(memberName, null, null); - logicalDependencies.add(ldepFile) - }else { - // debug-output - // println("$indentationMsg!* Skipped redundant analysis. $file was already or will be procceed soon.") - } - } - - if(logicalDependencies.size != 0) { - - // iterate over collections - metadataStore.getCollections().each{ collection -> - String cName = collection.getName() - if(matchesPattern(cName,collectionMatcherPatterns)){ // find matching collection names - - def Set externalImpactList = collectionImpactsSetMap.get(cName) ?: new HashSet() - // query dbb web app for files with all logicalDependencies - def logicalImpactedFiles = metadataStore.getImpactedFiles([cName], logicalDependencies); - - logicalImpactedFiles.each{ logicalFile -> - if (props.verbose) println("$indentationMsg Potential external impact found ${logicalFile.getLname()} (${logicalFile.getFile()}) in collection ${cName} ") - def impactRecord = "${logicalFile.getLname()} \t ${logicalFile.getFile()} \t ${cName}" - externalImpactList.add(impactRecord) - impactedFiles.add(logicalFile.getFile()) - } - // adding updated record - collectionImpactsSetMap.put(cName, externalImpactList) - - } - else{ - // debug-output - //if (props.verbose) println("$cName does not match pattern: $collectionMatcherPatterns") - } - } - } - else { - // debug-output - //if (props.verbose) println("Empty fileList") - } - return [ - collectionImpactsSetMap, - impactedFiles - ] -} - def updateCollection(changedFiles, deletedFiles, renamedFiles) { if (!MetadataStoreFactory.metadataStoreExists()) { @@ -792,7 +507,7 @@ def updateCollection(changedFiles, deletedFiles, renamedFiles) { if (props.verbose) println "** Updating collections ${props.applicationCollectionName} and ${props.applicationOutputsCollectionName}" //def scanner = new DependencyScanner() List logicalFiles = new ArrayList() - List excludeMatchers = createPathMatcherPattern(props.excludeFileList) + List excludeMatchers = buildUtils.createPathMatcherPattern(props.excludeFileList) verifyCollections() @@ -823,7 +538,7 @@ def updateCollection(changedFiles, deletedFiles, renamedFiles) { changedFiles.each { file -> // make sure file is not an excluded file - if ( new File("${props.workspace}/${file}").exists() && !matches(file, excludeMatchers)) { + if ( new File("${props.workspace}/${file}").exists() && !buildUtils.matches(file, excludeMatchers)) { // files in a collection are stored as relative paths from a source directory if (props.verbose) println "*** Scanning file $file (${props.workspace}/${file})" @@ -1037,17 +752,6 @@ def fixGitDiffPath(String file, String dir, boolean mustExist, mode) { return [defaultValue, null] } -def matches(String file, List pathMatchers) { - def result = pathMatchers.any { matcher -> - Path path = FileSystems.getDefault().getPath(file); - if ( matcher.matches(path) ) - { - return true - } - } - return result -} - /** * shouldCalculateImpacts * @@ -1056,60 +760,14 @@ def matches(String file, List pathMatchers) { */ def boolean shouldCalculateImpacts(String changedFile){ // retrieve Pathmaters from property and check - List nonImpactingFiles = createPathMatcherPattern(props.skipImpactCalculationList) - onskipImpactCalculationList = matches(changedFile, nonImpactingFiles) + List nonImpactingFiles = buildUtils.createPathMatcherPattern(props.skipImpactCalculationList) + onskipImpactCalculationList = buildUtils.matches(changedFile, nonImpactingFiles) // return false if changedFile found in skipImpactCalculationList if (onskipImpactCalculationList) return false return true //default } -/** - * createPathMatcherPattern - * Generic method to build PathMatcher from a build property - */ - -def createPathMatcherPattern(String property) { - List pathMatchers = new ArrayList() - if (property) { - property.split(',').each{ filePattern -> - if (!filePattern.startsWith('glob:') || !filePattern.startsWith('regex:')) - filePattern = "glob:$filePattern" - PathMatcher matcher = FileSystems.getDefault().getPathMatcher(filePattern) - pathMatchers.add(matcher) - } - } - return pathMatchers -} - -/** - * create List of Regex Patterns - */ - -def createMatcherPatterns(String property) { - List patterns = new ArrayList() - if (property) { - property.split(',').each{ patternString -> - Pattern pattern = Pattern.compile(patternString); - patterns.add(pattern) - } - } - return patterns -} - -/** - * match a String against a list of patterns - */ -def matchesPattern(String name, List patterns) { - def result = patterns.any { pattern -> - if (pattern.matcher(name).matches()) - { - return true - } - } - return result -} - /** * createPropertyDependency * method to add a dependency to a property key diff --git a/utilities/ReportingUtilities.groovy b/utilities/ReportingUtilities.groovy new file mode 100644 index 00000000..9aedde5f --- /dev/null +++ b/utilities/ReportingUtilities.groovy @@ -0,0 +1,344 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import java.nio.file.PathMatcher +import java.util.regex.* +import com.ibm.dbb.build.* +import com.ibm.dbb.dependency.* +import com.ibm.dbb.metadata.* +import groovy.transform.* +import java.net.URLEncoder + +// define script properties +@Field BuildProperties props = BuildProperties.getInstance() +@Field def gitUtils= loadScript(new File("GitUtilities.groovy")) +@Field def buildUtils= loadScript(new File("BuildUtilities.groovy")) +@Field def impactUtils= loadScript(new File("ImpactUtilities.groovy")) + +/** + * This utilities script is a collection of methods for the reporting + * capabilities of zAppBuild - for details see docs/REPORTS.md + * + * This includes + * + * - Report external impacted files (impacted logical files in other collections) + * - Report concurrent changes to document changes in other configurations within the repository + * + */ + + +// Methods for reporting external impacted files + +/** + * Method to query the DBB collections with a list of files + * Configured through reportExternalImpacts* build properties + */ + +def reportExternalImpacts(Set changedFiles){ + // query external collections to produce externalImpactList + + Map collectionImpactsSetMap = new HashMap() // + Set impactedFiles = new HashSet() + + List externalImpactReportingList = new ArrayList() + + if (props.verbose) println("*** Running external impact analysis with file filter ${props.reportExternalImpactsAnalysisFileFilter} and collection patterns ${props.reportExternalImpactsCollectionPatterns} with analysis mode ${props.reportExternalImpactsAnalysisDepths}") + + try { + + if (props.reportExternalImpactsAnalysisDepths == "simple" || props.reportExternalImpactsAnalysisDepths == "deep"){ + + // get directly impacted candidates first + if (props.verbose) println("*** Running external impact analysis for files ") + + // collect list changes files for which the analysis should be performed + changedFiles.each{ changedFile -> + + List fileMatchers = buildUtils.createPathMatcherPattern(props.reportExternalImpactsAnalysisFileFilter) + + // check that file is on reportExternalImpactsAnalysisFileFilter + if(buildUtils.matches(changedFile, fileMatchers)){ + + // get directly impacted candidates first + if (props.verbose) println(" $changedFile ") + externalImpactReportingList.add(changedFile) + + } else { + if (props.verbose) println("*** Analysis and reporting has been skipped for changed file $changedFile due to build framework configuration (see configuration of build property reportExternalImpactsAnalysisFileFilter)") + } + } + + if (externalImpactReportingList.size() != 0) { + + // calculate impacted files and write the report + def List logicalImpactedFilesCollections = calculateLogicalImpactedFiles(externalImpactReportingList, changedFiles, "buildSet") + writeExternalImpactReports(logicalImpactedFilesCollections, "***") + + // calculate impacted files and write the report, this performs the second level of impact analysis + if (props.reportExternalImpactsAnalysisDepths == "deep") { + + if (props.verbose) println("**** Running external impact analysis for identified external impacted files as dependent files of the initial set. ") + externalImpactReportingList.clear() + logicalImpactedFilesCollections.each { logicalImpactedFilesCollection -> + logicalImpactedFilesCollection.getLogicalFiles().each{ logicalFile -> + def impactedFile = logicalFile.getFile() + if (props.verbose) println(" $impactedFile ") + externalImpactReportingList.add(impactedFile) + } + } + + logicalImpactedFilesCollections = calculateLogicalImpactedFiles(externalImpactReportingList, changedFiles, "impactSet") + writeExternalImpactReports(logicalImpactedFilesCollections, "****") + } + } + } + else { + println("*! build property reportExternalImpactsAnalysisDepths has an invalid value : ${props.reportExternalImpactsAnaylsisDepths} , valid: simple | deep") + } + + } catch (Exception e) { + println("*! (ReportingUtilities.reportExternalImpacts) Exception caught during reporting of external impacts. Build continues.") + println(e.getMessage()) + println(e.printStackTrace()) + } +} + +/** + * Used to inspect dbb collections for potential impacts, sub-method to reportExternalImpacts + * + * Returns a collection of the identified potential impacts + * + */ + +def calculateLogicalImpactedFiles(List fileList, Set changedFiles, String analysisMode) { + MetadataStore metadataStore = MetadataStoreFactory.getMetadataStore() + + // local matchers to inspect files and collections + List collectionMatcherPatterns = createMatcherPatterns(props.reportExternalImpactsCollectionPatterns) + + // local variables + List logicalDependencies = new ArrayList() + List logicalImpactedFilesCollections = new ArrayList() + + // will be returned + Set impactedFiles = new HashSet() + + // construct the logical dependencies from the build list and filter on those files + // that we will search for + + fileList.each{ file -> + // go after all the files passed in; assess the identified impacted files to skip analysis for files from an impactSet which are on the changed files + if(analysisMode.equals('buildSet') || (analysisMode.equals('impactSet') && !changedFiles.contains(file))){ + String memberName = CopyToPDS.createMemberName(file) + def ldepFile = new LogicalDependency(memberName, null, null); + logicalDependencies.add(ldepFile) + } + } + + if(logicalDependencies.size != 0) { + + // get all collections which match pattern + List selectedCollections = new ArrayList() + metadataStore.getCollections().each{ it -> + cName = it.getName() + if (matchesPattern(cName,collectionMatcherPatterns)) selectedCollections.add(cName) + } + + // run query + logicalImpactedFilesCollections = metadataStore.getImpactedFiles(selectedCollections, logicalDependencies); + } + return logicalImpactedFilesCollections +} + + + +/** + * Generate the report files of the external impacts and write them to the build output directory + * + */ +def writeExternalImpactReports(List logicalImpactedFilesCollections, String indentationMsg) { + + // generate reports by collection / application + logicalImpactedFilesCollections.each{ collectionImpacts -> + + def List logicalImpactedFiles = collectionImpacts.getLogicalFiles() + def collectionName = collectionImpacts.getName() + + String encodedFileName = URLEncoder.encode("externalImpacts_${collectionName}.log", "UTF-8") + String impactListFileLoc = "${props.buildOutDir}/${encodedFileName}" + if (props.verbose) println("*** Writing report of external impacts to file $impactListFileLoc") + File impactListFile = new File(impactListFileLoc) + String enc = props.logEncoding ?: 'IBM-1047' + impactListFile.withWriter(enc) { writer -> + + // write message for each file + logicalImpactedFiles.each{ logicalFile -> + if (props.verbose) println("$indentationMsg Potential external impact found ${logicalFile.getLname()} (${logicalFile.getFile()}) in collection ${collectionName} ") + def impactRecord = "${logicalFile.getLname()} \t ${logicalFile.getFile()} \t ${collectionName}" + writer.write("$impactRecord\n") + } + } + } +} + +// Methods for reporting concurrent changes + +/** + * Method to calculate and report the changes between the current configuration and concurrent configurations; + * leverages the existing infrastructure to calculateChangedFiles - in this case for concurrent configs. + * + * Invokes method generateConcurrentChangesReports to produce the reports + * + * @param buildSet + * + */ +def calculateConcurrentChanges(Set buildSet) { + + // initialize patterns + List gitRefMatcherPatterns = createMatcherPatterns(props.reportConcurrentChangesGitBranchReferencePatterns) + + // obtain all current remote branches + // TODO: Handle / Exclude branches from other repositories + Set remoteBranches = new HashSet() + props.applicationSrcDirs.split(",").each { dir -> + dir = buildUtils.getAbsolutePath(dir) + remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) + } + + // Run analysis for each remoteBranch, which matches the configured criteria + remoteBranches.each { gitReference -> + + if (matchesPattern(gitReference,gitRefMatcherPatterns) && !gitReference.equals(props.applicationCurrentBranch)){ + + Set concurrentChangedFiles = new HashSet() + Set concurrentRenamedFiles = new HashSet() + Set concurrentDeletedFiles = new HashSet() + Set concurrentBuildProperties = new HashSet() + + if (props.verbose) println "*** Analysing and validating changes for branch : $gitReference" + + (concurrentChangedFiles, concurrentRenamedFiles, concurrentDeletedFiles, concurrentBuildProperties) = impactUtils.calculateChangedFiles(null, true, gitReference) + + // generate reports and verify for intersects + generateConcurrentChangesReports(buildSet, concurrentChangedFiles, concurrentRenamedFiles, concurrentDeletedFiles, gitReference) + + } + } + + } + +/* + * Method to generate the Concurrent Changes reports and validate if the current build list intersects with concurrent changes + */ + +def generateConcurrentChangesReports(Set buildList, Set concurrentChangedFiles, Set concurrentRenamedFiles, Set concurrentDeletedFiles, String gitReference){ + String concurrentChangesReportLoc = "${props.buildOutDir}/report_concurrentChanges.txt" + + File concurrentChangesReportFile = new File(concurrentChangesReportLoc) + String enc = props.logEncoding ?: 'IBM-1047' + concurrentChangesReportFile.withWriterAppend(enc) { writer -> + + if (!(concurrentChangedFiles.size() == 0 && concurrentRenamedFiles.size() == 0 && concurrentDeletedFiles.size() == 0)) { + + if (props.verbose) println("** Writing report of concurrent changes to $concurrentChangesReportLoc for configuration $gitReference") + + writer.write("\n=============================================== \n") + writer.write("** Report for configuration: $gitReference \n") + writer.write("========\n") + + if (concurrentChangedFiles.size() != 0) { + writer.write("** Changed Files \n") + concurrentChangedFiles.each { file -> + if (props.verbose) println " Changed: ${file}" + if (buildList.contains(file)) { + writer.write("* $file is changed and intersects with the current build list.\n") + String msg = "*!! $file is changed on branch $gitReference and intersects with the current build list." + println msg + + // update build result + if (props.reportConcurrentChangesIntersectionFailsBuild && props.reportConcurrentChangesIntersectionFailsBuild.toBoolean()) { + props.error = "true" + buildUtils.updateBuildResult(errorMsg:msg) + } else { + buildUtils.updateBuildResult(warningMsg:msg) + } + } + else + writer.write(" $file\n") + } + } + + if (concurrentRenamedFiles.size() != 0) { + writer.write("** Renamed Files \n") + concurrentRenamedFiles.each { file -> + if (props.verbose) println " Renamed: ${file}" + if (buildList.contains(file)) { + writer.write("* $file got renamed and intersects with the current build list.\n") + String msg = "*!! $file is renamed on branch $gitReference and intersects with the current build list." + println msg + + // update build result + if (props.reportConcurrentChangesIntersectionFailsBuild && props.reportConcurrentChangesIntersectionFailsBuild.toBoolean()) { + props.error = "true" + buildUtils.updateBuildResult(errorMsg:msg) + } else { + buildUtils.updateBuildResult(warningMsg:msg) + } + } + else + writer.write(" $file\n") + } + } + + if (concurrentDeletedFiles.size() != 0) { + writer.write("** Deleted Files \n") + concurrentDeletedFiles.each { file -> + if (props.verbose) println " Deleted: ${file}" + if (buildList.contains(file)) { + writer.write("* $file is deleted and intersects with the current build list.\n") + String msg = "*!! $file is deleted on branch $gitReference and intersects with the current build list." + println msg + + // update build result + if (props.reportConcurrentChangesIntersectionFailsBuild && props.reportConcurrentChangesIntersectionFailsBuild.toBoolean()) { + props.error = "true" + buildUtils.updateBuildResult(errorMsg:msg) + } else { + buildUtils.updateBuildResult(warningMsg:msg) + } + } + else + writer.write(" $file\n") + } + } + } + } +} + +// Internal matcher methods + +/** + * create List of Regex Patterns + */ + +def createMatcherPatterns(String property) { + List patterns = new ArrayList() + if (property) { + property.split(',').each{ patternString -> + Pattern pattern = Pattern.compile(patternString); + patterns.add(pattern) + } + } + return patterns +} + +/** +* match a String against a list of patterns +*/ +def matchesPattern(String name, List patterns) { + def result = patterns.any { pattern -> + if (pattern.matcher(name).matches()) + { + return true + } + } + return result +}