From bcc9e6d49d2b6b950e9838a50570b06e2b71ca41 Mon Sep 17 00:00:00 2001 From: Gerald Mitchell <55747427+GeraldMit@users.noreply.github.com> Date: Thu, 31 Mar 2022 12:35:11 -0400 Subject: [PATCH 01/14] Update README.md (#204) fix spelling --- utilities/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utilities/README.md b/utilities/README.md index 98081d7f..ca8234e9 100644 --- a/utilities/README.md +++ b/utilities/README.md @@ -4,8 +4,8 @@ This folder contains common utilty files used by the zAppBuild `build.groovy` sc File | Description --- | --- ADMIN.pw | Encrypted password file for password "ADMIN" which is the default ID and password for the DBB Web Application Liberty server. Convenient for initial set-up and testing of DBB functionality. For more information on creating password files see the documentation for the [Repository Client](https://www.ibm.com/support/knowledgecenter/SS6T76_1.0.4/buildresult.html#repository-client) in the DBB Knowledge Center. -BuidUtilities.groovy | Common build utilty methods. +BuildUtilities.groovy | Common build utility methods. GitUtilities.groovy | Git command methods. ImpactUtilities.groovy | Methods used for ImpactBuilds. BindUtilities.groovy | Use for the DB2 binding. -ScannerUtilities.groovy | Returns custom scanners. \ No newline at end of file +ScannerUtilities.groovy | Returns custom scanners. From ab5cc01f61115729ba55bbdfd64a1bbe4bfe9a32 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 7 Apr 2022 08:31:42 +0200 Subject: [PATCH 02/14] Leverage new SearchPathDependencyResolver and SearchPathImpactFinder API (#201) * Leverage new SearchPathDependencyResolver and SearchPathImpactFinder API * Update DBB Toolkit version validation --- build-conf/build.properties | 2 +- languages/Assembler.groovy | 26 +++- languages/Cobol.groovy | 23 +++- languages/LinkEdit.groovy | 13 +- languages/PLI.groovy | 24 +++- languages/REXX.groovy | 25 +++- languages/ZunitConfig.groovy | 25 ++-- .../application-conf/Cobol.properties | 5 + .../application-conf/PLI.properties | 78 ------------ .../application-conf/README.md | 10 +- .../application-conf/application.properties | 77 ++++++++++-- samples/application-conf/Assembler.properties | 6 + samples/application-conf/Cobol.properties | 6 + samples/application-conf/PLI.properties | 6 + samples/application-conf/README.md | 20 ++- samples/application-conf/REXX.properties | 6 + .../application-conf/ZunitConfig.properties | 6 + .../application-conf/application.properties | 118 +++++++++++++++++- utilities/BuildUtilities.groovy | 48 ++++--- utilities/ImpactUtilities.groovy | 47 ++++--- utilities/ResolverUtilities.groovy | 66 ++++++++++ 21 files changed, 478 insertions(+), 159 deletions(-) delete mode 100644 samples/MortgageApplication/application-conf/PLI.properties create mode 100644 utilities/ResolverUtilities.groovy diff --git a/build-conf/build.properties b/build-conf/build.properties index b5fe9076..e86183e0 100644 --- a/build-conf/build.properties +++ b/build-conf/build.properties @@ -45,7 +45,7 @@ applicationConfRootDir= # # Minimum required DBB ToolkitVersion to run this version of zAppBuild # Build initialization process validates the DBB Toolkit Version in use and matches that against this setting -requiredDBBToolkitVersion=1.0.0 +requiredDBBToolkitVersion=1.0.3 # # Comma separated list of required build properties for zAppBuild/build.groovy diff --git a/languages/Assembler.groovy b/languages/Assembler.groovy index 87c96e56..29251f0b 100644 --- a/languages/Assembler.groovy +++ b/languages/Assembler.groovy @@ -13,6 +13,11 @@ import com.ibm.dbb.build.report.records.* @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) @Field RepositoryClient repositoryClient +@Field def resolverUtils +// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2 +if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))} + println("** Building files mapped to ${this.class.getName()}.groovy script") // verify required build properties @@ -28,15 +33,24 @@ List sortedList = buildUtils.sortBuildList(argMap.buildList, 'assembler_ sortedList.each { buildFile -> println "*** Building file $buildFile" - // copy build file to input data set + // configure dependency resolution and create logical file + def dependencyResolver + LogicalFile logicalFile + + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.assembler_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver + String dependencySearch = props.getFileProperty('assembler_dependencySearch', buildFile) + dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + } else { // use deprecated DependencyResolver + String rules = props.getFileProperty('assembler_resolutionRules', buildFile) + dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + logicalFile = dependencyResolver.getLogicalFile() + } + // copy build file and dependency files to data sets - String rules = props.getFileProperty('assembler_resolutionRules', buildFile) - String assembler_srcPDS = props.getFileProperty('assembler_srcPDS', buildFile) - DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) - buildUtils.copySourceFiles(buildFile, assembler_srcPDS, 'assembler_dependenciesDatasetMapping', null ,dependencyResolver) + buildUtils.copySourceFiles(buildFile, props.assembler_srcPDS, 'assembler_dependenciesDatasetMapping', null ,dependencyResolver) // create mvs commands - LogicalFile logicalFile = dependencyResolver.getLogicalFile() String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.asm.log") if (logFile.exists()) diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 1a6fa202..55c56792 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -15,6 +15,11 @@ import com.ibm.dbb.build.report.records.* @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) @Field RepositoryClient repositoryClient +@Field def resolverUtils +// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2 +if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))} + println("** Building files mapped to ${this.class.getName()}.groovy script") // verify required build properties @@ -39,16 +44,28 @@ sortedList.each { buildFile -> // Check if this a testcase isZUnitTestCase = (props.getFileProperty('cobol_testcase', buildFile).equals('true')) ? true : false + // configure dependency resolution and create logical file + def dependencyResolver + LogicalFile logicalFile + + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.cobol_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver + String dependencySearch = props.getFileProperty('cobol_dependencySearch', buildFile) + dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + + } else { // use deprecated DependencyResolver + String rules = props.getFileProperty('cobol_resolutionRules', buildFile) + dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + logicalFile = dependencyResolver.getLogicalFile() + } + // copy build file and dependency files to data sets - String rules = props.getFileProperty('cobol_resolutionRules', buildFile) - DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) if(isZUnitTestCase){ buildUtils.copySourceFiles(buildFile, props.cobol_testcase_srcPDS, null, null, null) }else{ buildUtils.copySourceFiles(buildFile, props.cobol_srcPDS, 'cobol_dependenciesDatasetMapping', props.cobol_dependenciesAlternativeLibraryNameMapping, dependencyResolver) } // create mvs commands - LogicalFile logicalFile = dependencyResolver.getLogicalFile() String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.cobol.log") if (logFile.exists()) diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index 13904ef5..9905efde 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -30,10 +30,17 @@ sortedList.each { buildFile -> buildUtils.copySourceFiles(buildFile, props.linkedit_srcPDS, null, null, null) // create mvs commands - String rules = props.getFileProperty('linkedit_resolutionRules', buildFile) - DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) - LogicalFile logicalFile = dependencyResolver.getLogicalFile() + LogicalFile logicalFile + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean()) { // use new SearchPathDependencyResolver + logicalFile = SearchPathDependencyResolver.getLogicalFile(buildFile,props.workspace) + } + else { // use deprecated DependencyResolver API + DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, null) + logicalFile = dependencyResolver.getLogicalFile() + } + String member = CopyToPDS.createMemberName(buildFile) + File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.linkedit.log") if (logFile.exists()) logFile.delete() diff --git a/languages/PLI.groovy b/languages/PLI.groovy index 61502cc2..aee55b9c 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -14,6 +14,11 @@ import com.ibm.dbb.build.report.records.* @Field def impactUtils= loadScript(new File("${props.zAppBuildDir}/utilities/ImpactUtilities.groovy")) @Field RepositoryClient repositoryClient +@Field def resolverUtils +// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2 +if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))} + println("** Building files mapped to ${this.class.getName()}.groovy script") // verify required build properties @@ -37,11 +42,21 @@ sortedList.each { buildFile -> // Check if this a testcase isZUnitTestCase = (props.getFileProperty('pli_testcase', buildFile).equals('true')) ? true : false - // copy build file to input data set - // copy build file and dependency files to data sets - String rules = props.getFileProperty('pli_resolutionRules', buildFile) - DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + // configure dependency resolution and create logical file + def dependencyResolver + LogicalFile logicalFile + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.pli_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver + String dependencySearch = props.getFileProperty('pli_dependencySearch', buildFile) + dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + } else { // use deprecated DependencyResolver + String rules = props.getFileProperty('pli_resolutionRules', buildFile) + dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + logicalFile = dependencyResolver.getLogicalFile() + } + + // copy build file and dependency files to data sets if(isZUnitTestCase){ buildUtils.copySourceFiles(buildFile, props.pli_testcase_srcPDS, null, null, null) }else{ @@ -49,7 +64,6 @@ sortedList.each { buildFile -> } // create mvs commands - LogicalFile logicalFile = dependencyResolver.getLogicalFile() String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.pli.log") if (logFile.exists()) diff --git a/languages/REXX.groovy b/languages/REXX.groovy index 3a9c7ec9..0a4b5fd7 100644 --- a/languages/REXX.groovy +++ b/languages/REXX.groovy @@ -9,6 +9,12 @@ import com.ibm.dbb.repository.* @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) @Field RepositoryClient repositoryClient +@Field def resolverUtils +// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2 +if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))} + + println("** Building files mapped to ${this.class.getName()}.groovy script") // verify required build properties @@ -25,12 +31,25 @@ List sortedList = buildUtils.sortBuildList(argMap.buildList, 'rexx_fileB sortedList.each { buildFile -> println "*** Building file $buildFile" + + // configure dependency resolution and create logical file + def dependencyResolver + LogicalFile logicalFile + + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.rexx_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver + String dependencySearch = props.getFileProperty('rexx_dependencySearch', buildFile) + dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + logicalFile = resolverUtils.createLogicalFile(dependencyResolver, buildFile) + } else { // use deprecated DependencyResolver + String rules = props.getFileProperty('rexx_resolutionRules', buildFile) + dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + logicalFile = dependencyResolver.getLogicalFile() + } + // copy build file and dependency files to data sets - String rules = props.getFileProperty('rexx_resolutionRules', buildFile) - DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) buildUtils.copySourceFiles(buildFile, props.rexx_srcPDS, 'rexx_dependenciesDatasetMapping', null, dependencyResolver) + // create mvs commands - LogicalFile logicalFile = dependencyResolver.getLogicalFile() String member = CopyToPDS.createMemberName(buildFile) File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.REXX.log") if (logFile.exists()) diff --git a/languages/ZunitConfig.groovy b/languages/ZunitConfig.groovy index 8fd76dbb..d8756412 100644 --- a/languages/ZunitConfig.groovy +++ b/languages/ZunitConfig.groovy @@ -12,6 +12,11 @@ import groovy.transform.* @Field def bindUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BindUtilities.groovy")) @Field RepositoryClient repositoryClient +@Field def resolverUtils +// Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2 +if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("${props.zAppBuildDir}/utilities/ResolverUtilities.groovy"))} + println("** Building files mapped to ${this.class.getName()}.groovy script") // verify required build properties @@ -31,19 +36,25 @@ buildUtils.createLanguageDatasets(langQualifier) File logFile = new File("${props.buildOutDir}/${member}.zunit.jcl.log") File reportLogFile = new File("${props.buildOutDir}/${member}.zunit.report.log") + // configure dependency resolution + def dependencyResolver + + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.zunit_dependencySearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver + String dependencySearch = props.getFileProperty('zunit_dependencySearch', buildFile) + dependencyResolver = resolverUtils.createSearchPathDependencyResolver(dependencySearch) + } else { // use deprecated DependencyResolver + String rules = props.getFileProperty('zunit_resolutionRules', buildFile) + dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + } + // copy build file and dependency files to data sets - String rules = props.getFileProperty('zunit_resolutionRules', buildFile) - - DependencyResolver dependencyResolver = buildUtils.createDependencyResolver(buildFile, rules) + buildUtils.copySourceFiles(buildUtils.getAbsolutePath(buildFile), props.zunit_bzucfgPDS, 'zunit_dependenciesDatasetMapping', null, dependencyResolver) // Parse the playback from the bzucfg file Boolean hasPlayback = false String playback (hasPlayback, playback) = getPlaybackFile(buildFile); - - // Upload BZUCFG file to a BZUCFG Dataset - buildUtils.copySourceFiles(buildUtils.getAbsolutePath(buildFile), props.zunit_bzucfgPDS, 'zunit_dependenciesDatasetMapping', null, dependencyResolver) - + // Create JCLExec String String jobcard = props.jobCard.replace("\\n", "\n") String jcl = jobcard diff --git a/samples/MortgageApplication/application-conf/Cobol.properties b/samples/MortgageApplication/application-conf/Cobol.properties index 88f5a03c..d918b3cb 100644 --- a/samples/MortgageApplication/application-conf/Cobol.properties +++ b/samples/MortgageApplication/application-conf/Cobol.properties @@ -10,6 +10,11 @@ cobol_fileBuildRank= # Rules defined in application.properties cobol_resolutionRules=[${copybookRule}] +# +# COBOL dependencySearch configuration +# searchPath defined in application.properties +cobol_dependencySearch=${copybookSearch} + # # default COBOL compiler version # can be overridden by file properties diff --git a/samples/MortgageApplication/application-conf/PLI.properties b/samples/MortgageApplication/application-conf/PLI.properties deleted file mode 100644 index f6e9555a..00000000 --- a/samples/MortgageApplication/application-conf/PLI.properties +++ /dev/null @@ -1,78 +0,0 @@ -# Application properties used by zAppBuild/language/PLI.groovy - -# -# default PLI program build rank - used to sort language build file list -# leave empty - overridden by file properties if sorting needed -pli_fileBuildRank= - -# -# PLI dependency resolution rules -# Rules defined in rules.properties -pli_resolutionRules=[${plincRule}] - -# -# default PLI compiler version -# can be overridden by file properties -pli_compilerVersion=V52 - -# -# default PLI maximum RCs allowed -# can be overridden by file properties -pli_compileMaxRC=4 -pli_linkEditMaxRC=0 - -# -# lists of properties which should cause a rebuild after being changed -pli_impactPropertyList=pli_compilerVersion,pli_compileParms -pli_impactPropertyListCICS=pli_compileCICSParms -pli_impactPropertyListSQL=pli_compileSQLParms - -# -# default PLI compiler parameters -# can be overridden by file properties -pli_compileParms=PP(INCLUDE('ID(++INCLUDE)')) -pli_compileCICSParms=SYSTEM(CICS),PP(MACRO,CICS) -pli_compileSQLParms=PP(SQL) -pli_compileErrorPrefixParms=XINFO(XML) -pli_compileDebugParms=TEST - -# -# default LinkEdit parameters -# can be overridden by file properties -pli_linkEditParms=MAP,RENT,COMPAT(PM5) -pli_linkEditParms=DYNAM(DLL) :: **/testcase/*.pli -# -# execute link edit step -# can be overridden by file properties -pli_linkEdit=true - -# -# store abbrev git hash in ssi field -# available for buildTypes impactBuild, mergeBuild and fullBuild -# can be overridden by file properties -pli_storeSSI=true - -# -# default deployType -pli_deployType=LOAD - -# -# deployType for build files with isCICS=true -pli_deployTypeCICS=CICSLOAD - -# -# deployType for build files with isDLI=true -pli_deployTypeDLI=IMSLOAD - -# -# scan link edit load module for link dependencies -# can be overridden by file properties -pli_scanLoadModule=true - -# -# additional libraries for compile SYSLIB concatenation, comma-separated -pli_compileSyslibConcatenation= - -# -# additional libraries for linkEdit SYSLIB concatenation, comma-separated -pli_linkEditSyslibConcatenation= diff --git a/samples/MortgageApplication/application-conf/README.md b/samples/MortgageApplication/application-conf/README.md index 350f9e1a..81e67ae9 100644 --- a/samples/MortgageApplication/application-conf/README.md +++ b/samples/MortgageApplication/application-conf/README.md @@ -20,7 +20,12 @@ gitRepositoryURL | git repository URL of the application repository to establis excludeFileList | Files to exclude when scanning or running full build. skipImpactCalculationList | Files for which the impact analysis should be skipped in impact build jobCard | JOBCARD for JCL execs -impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. +useSearchConfiguration | Flag to define which DBB API is used for dependency and impact analysis. `false` uses DependencyResolver and ImpactResolver APIs, while `true` leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 +resolveSubsystems | boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile +impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. ** deprecated ** Please consider moving to new SearchPathDepedencyAPI leveraging `impactSearch` configuration. +impactSearch | Impact finder resolution search configuration leveraging the SearchPathImpactFinder API. Sample configurations are inlcuded below, next to the previous rule definitions. + + ### file.properties Location of file properties, script mappings and file level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for column 'Overridable' in the tables below for build properties that can have file level property overrides. @@ -51,7 +56,8 @@ Application properties used by zAppBuild/language/Cobol.groovy Property | Description | Overridable --- | --- | --- cobol_fileBuildRank | Default Cobol program build rank. Used to sort Cobol build file sub-list. Leave empty. | true -cobol_resolutionRules | Cobol dependency resolution rules used to create a Cobol dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. | true +cobol_resolutionRules | Cobol dependency resolution rules used to create a Cobol dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true +cobol_dependencySearch | Cobol dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true cobol_compilerVersion | Default Cobol compiler version. | true cobol_compileMaxRC | Default compile maximum RC allowed. | true cobol_linkEditMaxRC | Default link edit maximum RC allowed. | true diff --git a/samples/MortgageApplication/application-conf/application.properties b/samples/MortgageApplication/application-conf/application.properties index 39c6efcf..659855f5 100644 --- a/samples/MortgageApplication/application-conf/application.properties +++ b/samples/MortgageApplication/application-conf/application.properties @@ -35,39 +35,98 @@ excludeFileList=.*,**/.*,**/*.properties,**/*.xml,**/*.groovy,**/*.json,**/*.md, # sample: skipImpactCalculationList=**/epsmtout.cpy,**/centralCopybooks/*.cpy skipImpactCalculationList= +# +# flag to define which DBB API is used for dependency and impact analysis +# +# false = uses the DependencyResolver and ImpactResolver APIs (default) +# please note that DependencyResolver and ImpactResolver APIs are deprecated +# configuration is based on the lang_resolutionRules and impactResolutionRules build properties +# true = leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 +# configuration is based on the lang_dependencySearch and impactSearch build properties +useSearchConfiguration=false + +# +# boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact +# the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile +# +# requires to use new APIs via useSearchConfiguration=true +# default:false +resolveSubsystems=false + # # Impact analysis resolution rules (JSON format) +# leverages the ImpactResolver API +# ** deprecated ** - see zAppBuild build property >impactSearch< impactResolutionRules=[${copybookRule},${bmsRule},${linkRule},${propertyRule}] +# +# Impact finder resolution search configuration +# leverages the SearchPathImpactFinder API +impactSearch=${copybookSearch}${bmsSearch}${linkSearch} + +# # Rule to locate Cobol copy books +# ** deprecated ** copybookRule = {"library": "SYSLIB", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/copybook"} \ ] \ } -# Rule to locate BMS maps +# +# copybookSearch +# searchPath to locate Cobol copybooks +# used in dependency resolution and impact analysis +# +# Please be as specific as possible when configuring the searchPath. +# Alternate configurations: +# +# dependency resolution from multiple repositories / multiple root folders: +# copybookSearch = search:${workspace}/?path=**/copybook/*.cpy +# +# dependency resolution across all directories in build workspace, but filtering on the file extension cpy: +# copybookSearch = search:${workspace}/?path=**/*.cpy +# dependency resolution across all directories in build workspace, but filtering on the file extension cpy and cobcpy: +# copybookSearch = search:${workspace}/?path=**/*.cpy;**/*.cobcpy +# +# dependency resolution in the application directory and a shared common copybook location: +# copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy;/u/build/common/copybooks/*.cpy +# +# More samples can be found at: https://www.ibm.com/docs/en/adfz/dbb/1.1.0?topic=scripts-how-manage-build-dependencies +# +copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy + +# +# Rule to locate BMS maps +# ** deprecated ** bmsRule = {"library": "SYSLIB", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/bms"} \ ] \ } -# Rule to locate COBOL programs and Link files for rebuilding statically linked load modules +# +# bmsSearch +# searchPath to locate impacted bms maps +# used only in impact analysis +bmsSearch = search:${workspace}/?path=${application}/bms/*.bms + +# Rule to locate COBOL programs and Link files for rebuilding statically linked load modules +# ** deprecated ** linkRule = {"category": "LINK", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/cobol"}, \ {"sourceDir": "${workspace}", "directory": "${application}/link"} \ ] \ } - + +# +# linkSearch +# searchPath to locate impacted programs or linkcards +# category LINK only; used only in impact analysis +linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl;${application}/link/*.lnk # Rule to detect DBB Build property changes +# ** deprecated ** propertyRule = {"category": "PROPERTY"} - - - - - - diff --git a/samples/application-conf/Assembler.properties b/samples/application-conf/Assembler.properties index 4f0b5f87..921d7826 100644 --- a/samples/application-conf/Assembler.properties +++ b/samples/application-conf/Assembler.properties @@ -34,8 +34,14 @@ assembler_impactPropertyListSQL=assembler_cicsprecompilerParms # # ASM Dependency resolution rules # Rules defined in app-properties file +# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch assembler_resolutionRules=[${maclibRule},${asmCopyRule}] +# +# ASM dependencySearch configuration +# searchPaths defined in app-properties file +assembler_dependencySearch=${asmMacroSearch}${asmCopySearch} + # # store abbrev git hash in ssi field # available for buildTypes impactBuild, mergeBuild and fullBuild diff --git a/samples/application-conf/Cobol.properties b/samples/application-conf/Cobol.properties index 76f03938..6ae40306 100644 --- a/samples/application-conf/Cobol.properties +++ b/samples/application-conf/Cobol.properties @@ -8,8 +8,14 @@ cobol_fileBuildRank= # # COBOL dependency resolution rules # Rules defined in application.properties +# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch cobol_resolutionRules=[${copybookRule}] +# +# COBOL dependencySearch configuration +# searchPath defined in application.properties +cobol_dependencySearch=${copybookSearch} + # # default COBOL compiler version # can be overridden by file properties diff --git a/samples/application-conf/PLI.properties b/samples/application-conf/PLI.properties index dd13e167..dba32378 100644 --- a/samples/application-conf/PLI.properties +++ b/samples/application-conf/PLI.properties @@ -8,8 +8,14 @@ pli_fileBuildRank= # # PLI dependency resolution rules # Rules defined in rules.properties +# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch pli_resolutionRules=[${plincRule}] +# +# PLI dependencySearch configuration +# searchPath defined in application.properties +pli_dependencySearch=${pliincludeSearch} + # # default PLI compiler version # can be overridden by file properties diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index bd683159..cd39a837 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -23,7 +23,10 @@ gitRepositoryURL | git repository URL of the application repository to establis excludeFileList | Files to exclude when scanning or running full build. | false skipImpactCalculationList | Files for which the impact analysis should be skipped in impact build | false jobCard | JOBCARD for JCL execs | false -impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. | true, recommended in file.properties +useSearchConfiguration | Flag to define which DBB API is used for dependency and impact analysis. `false` uses DependencyResolver and ImpactResolver APIs, while `true` leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 | false +resolveSubsystems | boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile | false +impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. ** deprecated ** Please consider moving to new SearchPathDepedencyAPI leveraging `impactSearch` configuration. | true, recommended in file.properties +impactSearch | Impact finder resolution search configuration leveraging the SearchPathImpactFinder API. Sample configurations are inlcuded below, next to the previous rule definitions. | true ### file.properties Location of file properties, script mappings and file level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for column 'Overridable' in the tables below for build properties that can have file level property overrides. @@ -66,7 +69,8 @@ assembler_linkEditMaxRC | Default link edit maximum RC allowed. | true assembler_impactPropertyList | List of build properties causing programs to rebuild when changed | false assembler_impactPropertyListCICS | List of CICS build properties causing programs to rebuild when changed | false assembler_impactPropertyListSQL | List of SQL build properties causing programs to rebuild when changed | false -assembler_resolutionRules | Assembler dependency resolution rules used to create a Assmebler dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. | true +assembler_resolutionRules | Assembler dependency resolution rules used to create a Assmebler dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true +assembler_dependencySearch | Assembler dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true assembler_storeSSI | Flag to store abbrev git hash in ssi field in link step | true assembler_deployType | default deployType for build output | true assembler_deployTypeCICS | deployType for build output for build files where isCICS=true | true @@ -97,7 +101,8 @@ Application properties used by zAppBuild/language/Cobol.groovy Property | Description | Overridable --- | --- | --- cobol_fileBuildRank | Default Cobol program build rank. Used to sort Cobol build file sub-list. Leave empty. | true -cobol_resolutionRules | Cobol dependency resolution rules used to create a Cobol dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. | true +cobol_resolutionRules | Cobol dependency resolution rules used to create a Cobol dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true +cobol_dependencySearch | Cobol dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true cobol_compilerVersion | Default Cobol compiler version. | true cobol_compileMaxRC | Default compile maximum RC allowed. | true cobol_linkEditMaxRC | Default link edit maximum RC allowed. | true @@ -142,7 +147,8 @@ Application properties used by zAppBuild/language/LinkEdit.groovy Property | Description | Overridable --- | --- | --- pli_fileBuildRank | Default PLI program build rank. Used to sort PLI program sub-list. Leave empty. | true -pli_resolutionRules | PLI dependency resolution rules used to create a PLI dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. | true +pli_resolutionRules | PLI dependency resolution rules used to create a PLI dependency resolver. Format is a JSON array of resolution rule property keys. Resolution rule properties are defined in `application-conf/application.properties`. ** deprecated ** | true +pli_dependencySearch | PLI dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true pli_compilerVersion | Default PLI compiler version. | true pli_compileMaxRC | Default compile maximum RC allowed. | true pli_linkEditMaxRC | Default link edit maximum RC allowed. | true @@ -240,7 +246,8 @@ Property | Description | Overridable zunit_maxPassRC | Default zUnit maximum RC allowed for a Pass. | true zunit_maxWarnRC | Default zUnit maximum RC allowed for a Warninig (everything beyond this value will Fail). | true zunit_playbackFileExtension | Default zUnit Playback File Extension. | true -zunit_resolutionRules | Default resolution rules for zUnit. | true +zunit_resolutionRules | Default resolution rules for zUnit. ** deprecated ** | true +zunit_dependencySearch | Default zUnit dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true zunit_bzuplayParms | Default options passed to the zUnit runner BZUPLAY | true zunit_userDebugSessionTestParm | Debug Tool Test parameter to initiate the debug session | true zunit_CodeCoverageHost | Headless Code Coverage Collector host (if not specified IDz will be used for reporting) | true @@ -254,7 +261,8 @@ Property | Description | Overridable --- | --- | --- rexx_compileMaxRC | Default compile maximum RC allowed. | true rexx_linkEditMaxRC | Default link edit maximum RC allowed. | true -rexx_resolutionRules | Default resolution rules for zUnit. | true +rexx_resolutionRules | Default resolution rules for zUnit. ** deprecated ** | true +rexx_dependencySearch | Default REXX dependencySearch configuration to configure the SearchPathDependencyResolver. Format is a concatenated string of searchPath configurations. Strings representing the SearchPaths defined in `application-conf/application.properties`. | true rexx_compileParms | Default base compile parameters. | true rexx_compiler | Default REXX compiler | true rexx_linkEdit | Flag indicating to execute the link edit step to produce a compiled rexx for the source file. | true diff --git a/samples/application-conf/REXX.properties b/samples/application-conf/REXX.properties index edca26c9..62e8db43 100644 --- a/samples/application-conf/REXX.properties +++ b/samples/application-conf/REXX.properties @@ -8,8 +8,14 @@ rexx_fileBuildRank= # # REXX dependency resolution rules # Rules defined in application.properties +# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch rexx_resolutionRules=[${rexxRule}] +# +# REXX dependencySearch configuration +# searchPath defined in application.properties +rexx_dependencySearch=${rexxCopySearch} + # # default REXX maximum RCs allowed # can be overridden by file properties diff --git a/samples/application-conf/ZunitConfig.properties b/samples/application-conf/ZunitConfig.properties index dbadabdd..0c1bb6a6 100755 --- a/samples/application-conf/ZunitConfig.properties +++ b/samples/application-conf/ZunitConfig.properties @@ -12,8 +12,14 @@ zunit_playbackFileExtension=plbck # # zUnit dependency resolution rules # Rules defined in application.properties +# ** deprecated **, please consider moving to new SearchPathDepedencyAPI leveraging zunit_dependencySearch zunit_resolutionRules=[${testcaseRule}] +# +# zUnit dependencySearch configuration +# searchPath defined in application.properties +zunit_dependencySearch=${zunitPlayfileSearch} + # # Options passed to the zUnit runner BZUPLAY # https://www.ibm.com/docs/en/zvtp/2.0?topic=playback-setup-execution-parameters diff --git a/samples/application-conf/application.properties b/samples/application-conf/application.properties index cb449fa5..aafed332 100644 --- a/samples/application-conf/application.properties +++ b/samples/application-conf/application.properties @@ -55,21 +55,68 @@ skipImpactCalculationList= # Example: jobCard=//RUNZUNIT JOB ,MSGCLASS=H,CLASS=A,NOTIFY=&SYSUID,REGION=0M jobCard= +# +# flag to define which DBB API is used for dependency and impact analysis +# +# false = uses the DependencyResolver and ImpactResolver APIs (default) +# please note that DependencyResolver and ImpactResolver APIs are deprecated +# configuration is based on the lang_resolutionRules and impactResolutionRules build properties +# true = leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 +# configuration is based on the lang_dependencySearch and impactSearch build properties +useSearchConfiguration=false +# +# boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact +# the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile +# +# requires to use new APIs via useSearchConfiguration=true +# default:false +resolveSubsystems=false # # Impact analysis resolution rules (JSON format). # Defaults to just looking for local application dependency folders +# ** deprecated ** - see zAppBuild build property >impactSearch< impactResolutionRules=[${copybookRule},${plincRule},${maclibRule},${asmCopyRule},${rexxRule},${linkRule},${testcaseRule},${testconfigRule},${testcasePgmRule},${propertyRule}] +# +# Impact finder resolution search configuration +# leverages the SearchPathImpactFinder API +impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${linkCardSearch}${zunitTestConfigSearch}${zunitApplicationPgmSearch} + # Rule to locate Cobol copy books. This rule defaults to the local copybook folder # in the main application folder. +# ** deprecated ** copybookRule = {"library": "SYSLIB", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/copybook"} \ ] \ } +# +# copybookSearch +# searchPath to locate Cobol copybooks +# used in dependency resolution and impact analysis +# +# Please be as specific as possible when configuring the searchPath. +# Alternate configurations: +# +# dependency resolution from multiple repositories / multiple root folders: +# copybookSearch = search:${workspace}/?path=**/copybook/*.cpy +# +# dependency resolution across all directories in build workspace, but filtering on the file extension cpy: +# copybookSearch = search:${workspace}/?path=**/*.cpy +# dependency resolution across all directories in build workspace, but filtering on the file extension cpy and cobcpy: +# copybookSearch = search:${workspace}/?path=**/*.cpy;**/*.cobcpy +# +# dependency resolution in the application directory and a shared common copybook location: +# copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy;/u/build/common/copybooks/*.cpy +# +# More samples can be found at: https://www.ibm.com/docs/en/adfz/dbb/1.1.0?topic=scripts-how-manage-build-dependencies +# +copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy + + # Rule to locate PLI include files. This rule defaults to the local plinc folder # in the main application folder. plincRule = {"library": "SYSLIB", \ @@ -77,59 +124,128 @@ plincRule = {"library": "SYSLIB", \ {"sourceDir": "${workspace}", "directory": "${application}/plinc"} \ ] \ } +# +# pliincludeSearch +# searchPath to locate PLI include files +# used in dependency resolution and impact analysis +pliincludeSearch = search:${workspace}/?path=${application}/plinc/*.cpy # Rule to locate ASM macros. This rule defaults to the local maclib folder # in the main application folder. # Category filters on what you want to include during the scanner (i.e. excludes macro-def keyword) +# ** deprecated ** maclibRule = {"library": "SYSLIB", "category": "MACRO", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/maclib"} \ ] \ } +# asmMacroSearch +# searchPath to locate Assembler macro files +# used in dependency resolution and impact analysis +asmMacroSearch = search:[SYSLIB:MARCO]${workspace}/?path=${application}/maclib/*.mac + # Rule to locate ASM copybooks. This rule defaults to the local maclib folder # in the main application folder. +# ** deprecated ** asmCopyRule = {"library": "SYSLIB", "category": "COPY", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/maclib"} \ ] \ } +# asmCopySearch +# searchPath to locate Assembler copy files +# used in dependency resolution and impact analysis +asmCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/maclib/*.mac + +# +# Rule to locate BMS maps +# ** deprecated ** +bmsRule = {"library": "SYSLIB", \ + "searchPath": [ \ + {"sourceDir": "${workspace}", "directory": "${application}/bms"} \ + ] \ + } + +# +# bmsSearch +# searchPath to locate impacted bms maps +# used only in impact analysis +bmsSearch = search:${workspace}/?path=${application}/bms/*.bms + # Rule to locate REXX includes. This rule defaults to the local rexx folder # in the main application folder. +# ** deprecated ** rexxRule = {"library": "SYSLIB", "category": "COPY", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/rexx"} \ ] \ } +# +# rexxCopySearch +# searchPath to locate rexx copy +# used in dependency resolution and impact analysis +rexxCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/rexx/*.rexx + # Rule to locate Link files for rebuilding statically linked load modules + linkRule = {"category": "LINK", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/link"} \ ] \ } +# +# linkCardSearch +# searchPath to locate impacted linkcards +# category LINK only; used only in impact analysis +linkCardSearch = search:[:LINK]${workspace}/?path=${application}/link/*.lnk + # Rule to locate the zUnit test configuration file +# ** deprecated ** -> zunitTestConfigSearch testconfigRule = {"library": "SYSPROG", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/testcfg"} \ ] \ } +# zunitTestConfigSearch +# searchPath to locate zunit config files +# used in dependency resolution and impact analysis +zunitTestConfigSearch = search:[SYSPROG:]${workspace}/?path=${application}/testcfg/*.bzucfg + # Rule to locate the zUnit playback file +# ** deprecated ** -> zunitPlayfileSearch testcaseRule = {"library": "SYSPLAY", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/testplayfiles"} \ ] \ } +# +# zunitPlayfileSearch +# searchPath to locate zunit playback files +# used in dependency resolution +zunitPlayfileSearch = search:[SYSPLAY:]${workspace}/?path=${application}/testplayfiles/*.bzuplay + + # Rule to locate the zUnit test case program when the program or the bzucfg file changes +# ** deprecated ** -> zunitApplicationPgmSearch testcasePgmRule = {"category": "PROGRAMDEPENDENCY", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/cobol"} \ ] \ } +# +# zunitApplicationPgmSearch +# searchPath to locate impacted application programs +# see also build-conf/build.properties -> createTestcaseDependency +# used in impact analysis +zunitApplicationPgmSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli + # Rule to detect DBB Build property changes -propertyRule = {"category": "PROPERTY"} \ No newline at end of file +# ** deprecated ** +propertyRule = {"category": "PROPERTY"} diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index b8784175..27fc6b8f 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -16,8 +16,6 @@ import groovy.ant.* @Field HashSet copiedFileCache = new HashSet() @Field def gitUtils = loadScript(new File("GitUtilities.groovy")) - - /* * assertBuildProperties - verify that required build properties for a script exist */ @@ -93,7 +91,7 @@ def getFileSet(String dir, boolean relativePaths, String includeFileList, String * - DependencyResolver to resolve dependencies */ -def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMapping, String dependenciesAlternativeLibraryNameMapping, DependencyResolver dependencyResolver) { +def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMapping, String dependenciesAlternativeLibraryNameMapping, Object dependencyResolver) { // only copy the build file once if (!copiedFileCache.contains(buildFile)) { copiedFileCache.add(buildFile) @@ -117,10 +115,7 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap String lname = CopyToPDS.createMemberName(buildFile) String language = props.getFileProperty('dbb.DependencyScanner.languageHint', buildFile) ?: 'UNKN' LogicalFile lfile = new LogicalFile(lname, buildFile, language, depFileData.isCICS, depFileData.isSQL, depFileData.isDLI) - // save logical file to dependency resolver - if (dependencyResolver) - dependencyResolver.setLogicalFile(lfile) - + // get list of dependencies from userBuildDependencyFile List dependencyPaths = depFileData.dependencies @@ -160,25 +155,33 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap } else if (dependencyDatasetMapping && dependencyResolver) { // resolve the logical dependencies to physical files to copy to data sets - List physicalDependencies = dependencyResolver.resolve() - if (props.verbose) { - println "*** Resolution rules for $buildFile:" - - if (props.formatConsoleOutput && props.formatConsoleOutput.toBoolean()) { - printResolutionRules(dependencyResolver.getResolutionRules()) - } else { - dependencyResolver.getResolutionRules().each{ rule -> println rule } + List physicalDependencies + + if (dependencyResolver instanceof DependencyResolver) { // deprecated DependencyResolver + physicalDependencies = dependencyResolver.resolve() + if (props.verbose) { + println "*** Resolution rules for $buildFile:" + + if (props.formatConsoleOutput && props.formatConsoleOutput.toBoolean()) { + printResolutionRules(dependencyResolver.getResolutionRules()) + } else { + dependencyResolver.getResolutionRules().each{ rule -> println rule } + } } + } else if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("ResolverUtilities.groovy")) + physicalDependencies = resolverUtils.resolveDependencies(dependencyResolver, buildFile) } + if (props.verbose) println "*** Physical dependencies for $buildFile:" // Load property mapping containing the map of targetPDS and dependencyfile PropertyMappings dependenciesDatasetMapping = new PropertyMappings(dependencyDatasetMapping) - + if (physicalDependencies.size() != 0) { if (props.verbose && props.formatConsoleOutput && props.formatConsoleOutput.toBoolean()) { printPhysicalDependencies(physicalDependencies) - } + } } physicalDependencies.each { physicalDependency -> @@ -656,9 +659,14 @@ def assertDbbBuildToolkitVersion(String currentVersion, String requiredVersion){ assert currentVersionList.size() >= requiredVersionList.size() : "Version syntax does not match." // validate each label - currentVersionList.eachWithIndex{ it, i -> - if(requiredVersionList.size() >= i +1 ) assert (it as int) >= ((requiredVersionList[i]) as int) - } + Boolean foundValidVersion = false + currentVersionList.eachWithIndex{ label, i -> + if((requiredVersionList.size() >= i +1 ) && !(foundValidVersion.toBoolean() == true)) { + assert (label as int) >= ((requiredVersionList[i]) as int) : "Current DBB Toolkit Version $currentVersion does not meet the minimum required version $requiredVersion. EXIT." + if (label > requiredVersionList[i]) foundValidVersion = true + } + + } } catch(AssertionError e) { println "Current DBB Toolkit Version $currentVersion does not meet the minimum required version $requiredVersion. EXIT." diff --git a/utilities/ImpactUtilities.groovy b/utilities/ImpactUtilities.groovy index 9555dde4..659ab9bf 100644 --- a/utilities/ImpactUtilities.groovy +++ b/utilities/ImpactUtilities.groovy @@ -14,9 +14,15 @@ import java.util.regex.* @Field def gitUtils= loadScript(new File("GitUtilities.groovy")) @Field def buildUtils= loadScript(new File("BuildUtilities.groovy")) @Field String hashPrefix = ':githash:' +@Field def resolverUtils def createImpactBuildList(RepositoryClient repositoryClient) { + + // Conditionally load the ResolverUtilities.groovy which require at least DBB 1.1.2 + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { + resolverUtils = loadScript(new File("ResolverUtilities.groovy")) } + // local variables Set changedFiles = new HashSet() Set deletedFiles = new HashSet() @@ -63,26 +69,37 @@ def createImpactBuildList(RepositoryClient repositoryClient) { // perform impact analysis on changed file if (props.verbose) println "** Performing impact analysis on changed file $changedFile" - String impactResolutionRules = props.getFileProperty('impactResolutionRules', changedFile) - ImpactResolver impactResolver = createImpactResolver(changedFile, impactResolutionRules, repositoryClient) - - // get excludeListe + // get exclude list List excludeMatchers = createPathMatcherPattern(props.excludeFileList) + + // list of impacts + def impacts - // Print impactResolverConfiguration - if (props.verbose && props.formatConsoleOutput && props.formatConsoleOutput.toBoolean()) { - // print collection information - println(" " + "Collection".padRight(20) ) - println(" " + " ".padLeft(20,"-")) - impactResolver.getCollections().each{ collectionName -> - println(" " + collectionName) + if (props.useSearchConfiguration && props.useSearchConfiguration.toBoolean() && props.impactSearch && buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, "1.1.2")) { // use new SearchPathDependencyResolver + + String impactSearch = props.getFileProperty('impactSearch', changedFile) + impacts = resolverUtils.findImpactedFiles(impactSearch, changedFile, repositoryClient) + } + else { + String impactResolutionRules = props.getFileProperty('impactResolutionRules', changedFile) + ImpactResolver impactResolver = createImpactResolver(changedFile, impactResolutionRules, repositoryClient) + + // Print impactResolverConfiguration + if (props.verbose && props.formatConsoleOutput && props.formatConsoleOutput.toBoolean()) { + // print collection information + println(" " + "Collection".padRight(20) ) + println(" " + " ".padLeft(20,"-")) + impactResolver.getCollections().each{ collectionName -> + println(" " + collectionName) + } + // print impact resolution rule in table format + buildUtils.printResolutionRules(impactResolver.getResolutionRules()) } - // print impact resolution rule in table format - buildUtils.printResolutionRules(impactResolver.getResolutionRules()) + + // resolving impacts + impacts = impactResolver.resolve() } - // resolving impacts - def impacts = impactResolver.resolve() impacts.each { impact -> def impactFile = impact.getFile() if (props.verbose) println "** Found impacted file $impactFile" diff --git a/utilities/ResolverUtilities.groovy b/utilities/ResolverUtilities.groovy new file mode 100644 index 00000000..4dee5aef --- /dev/null +++ b/utilities/ResolverUtilities.groovy @@ -0,0 +1,66 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.dependency.* +import com.ibm.dbb.repository.* +import com.ibm.dbb.build.* +import groovy.transform.* + +@Field BuildProperties props = BuildProperties.getInstance() + +// Externalized Method to preserve backward compatibility with older DBB toolkit versions. +// TODO: Refactoring as soon as the deprecated API is dropped. + +/** + * Method to create the logical file using SearchPathDependencyResolver + * + * evaluates if it should resolve file flags for resolved dependencies + * + * @param spDependencyResolver + * @param buildFile + * @return + */ + +def createLogicalFile(SearchPathDependencyResolver spDependencyResolver, String buildFile) { + + LogicalFile logicalFile + + if (props.resolveSubsystems && props.resolveSubsystems.toBoolean()) // include resolved dependencies to define file flags of logicalFile + logicalFile = spDependencyResolver.resolveSubsystems(buildFile,props.workspace) + else + logicalFile = SearchPathDependencyResolver.getLogicalFile(buildFile,props.workspace) + + return logicalFile + +} + +/** + * + * @param dependencySearch + * @return SearchPathDependencyResolver + */ +def createSearchPathDependencyResolver(String dependencySearch) { + return new SearchPathDependencyResolver(dependencySearch) +} + +def findImpactedFiles(String impactSearch, String changedFile, RepositoryClient repositoryClient) { + + List collections = new ArrayList() + collections.add(props.applicationCollectionName) + collections.add(props.applicationOutputsCollectionName) + + if (props.verbose) + println ("*** Creating SearchPathImpactFinder with collections " + collections + " and impactSearch configuration " + impactSearch) + + def finder = new SearchPathImpactFinder(impactSearch, collections, repositoryClient) + + // Find all files impacted by the changed file + impacts = finder.findImpactedFiles(changedFile, props.workspace) + return impacts +} + +def resolveDependencies(SearchPathDependencyResolver dependencyResolver, String buildFile) { + if (props.verbose) { + println "*** Resolution rules for $buildFile:" + println dependencyResolver.getSearchPath() + } + return dependencyResolver.resolveDependencies(buildFile, props.workspace) +} \ No newline at end of file From 492c766f5e91b613b23864a6e39aea46572ba1ce Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Mon, 16 May 2022 07:31:52 +0200 Subject: [PATCH 03/14] Update build-conf/README.md (#210) --- build-conf/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-conf/README.md b/build-conf/README.md index 794c0645..afc04541 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -57,6 +57,7 @@ dbb.RepositoryClient.passwordFile | DBB configuration property for web applicati ### dependencyReport.properties Properties used by the impact utilities to generate a report of external impacted files +Property | Description --- | --- reportExternalImpacts | Flag to indicate if an *impactBuild* should analyze and report external impacted files in other collections ***Can be overridden by build.groovy option -re, --reportExternalImpacts*** reportExternalImpactsAnalysisDepths | Configuration of the analysis depths when performing impact analysis for external impacts (simple|deep) *** Can be overridden by application-conf *** @@ -284,4 +285,4 @@ transfer_srcPDS | Dataset of any type of source transfer_jclPDS | Sample dataset for JCL members transfer_xmlPDS | Sample dataset for xml members transfer_srcOptions | BPXWDYN creation options for creating 'source' type data sets -transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets \ No newline at end of file +transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets From 435c506f84949e7b6b955d4b0834a83a197b7e18 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 23 May 2022 07:21:29 +0200 Subject: [PATCH 04/14] documentation update - fix md syntax to correctly display build log in collapsed section in BUILD.md (#206) --- BUILD.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BUILD.md b/BUILD.md index 9ff13fce..7438506c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -249,6 +249,7 @@ groovyz dbb-zappbuild/build.groovy --workspace /var/dbb/dbb-zappbuild/samples -- ```
Build log + ``` ** Build start at 20210622.081915.019 ** Input args = /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --verbose /var/dbb/MortgageApplication/myBuildList.txt @@ -331,6 +332,7 @@ Cobol compiler parms for MortgageApplication/cobol/epscmort.cbl = LIB,CICS,SQL ** Total files processed : 2 ** Total build time : 36.978 seconds ``` +
### Perform Full Build to build all files @@ -577,6 +579,7 @@ groovyz dbb-zappbuild/build.groovy --workspace /var/dbb/dbb-zappbuild/samples -- ```
Build log + ``` ** Build start at 20210622.082942.029 ** Input args = /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --impactBuild --verbose From 9135790485ad6ac76d03923f503eeecb357f52e4 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 24 May 2022 12:06:56 +0200 Subject: [PATCH 05/14] Update impact resolution rule samples for static linkage scenarios (#215) * ensure consistency between link rules and link search configurations --- .../application-conf/application.properties | 22 ++++++-- .../application-conf/application.properties | 50 +++++++++++++++---- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/samples/MortgageApplication/application-conf/application.properties b/samples/MortgageApplication/application-conf/application.properties index 659855f5..085dd17a 100644 --- a/samples/MortgageApplication/application-conf/application.properties +++ b/samples/MortgageApplication/application-conf/application.properties @@ -115,16 +115,30 @@ bmsSearch = search:${workspace}/?path=${application}/bms/*.bms # ** deprecated ** linkRule = {"category": "LINK", \ "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/cobol"}, \ - {"sourceDir": "${workspace}", "directory": "${application}/link"} \ + {"sourceDir": "${workspace}", "directory": "${application}/cobol"} \ ] \ } # # linkSearch -# searchPath to locate impacted programs or linkcards +# +# searchPath to locate impacted linkcards or main programs after an included submodule is changed +# leverages the output collection, which has the dependency info from the executable # category LINK only; used only in impact analysis -linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl;${application}/link/*.lnk +# +# Additional samples: +# +# impact resolution across all directories in build workspace, but filtering on the file extension cbl: +# staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl +# +# impact resolution across all directories in build workspace, but filtering on the file extension cbl and pli (for cobol and pli submodules): +# staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl,**/*.pli +# +# Special case with Dependency Scanner Transfer Control Statement capturing turned on (default is off) +# the scanners detect a static call to the literal, which would need to turn into a new rule for CALL: +# staticCallSearch = search:[:CALL]${workspace}/?path=${application}/cobol/*.cbl +# +linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl # Rule to detect DBB Build property changes # ** deprecated ** diff --git a/samples/application-conf/application.properties b/samples/application-conf/application.properties index aafed332..912b15d8 100644 --- a/samples/application-conf/application.properties +++ b/samples/application-conf/application.properties @@ -82,10 +82,11 @@ impactResolutionRules=[${copybookRule},${plincRule},${maclibRule},${asmCopyRule} # # Impact finder resolution search configuration # leverages the SearchPathImpactFinder API -impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${linkCardSearch}${zunitTestConfigSearch}${zunitApplicationPgmSearch} +impactSearch=${copybookSearch}${pliincludeSearch}${bmsSearch}${linkSearch}${rexxCopySearch}${zunitTestConfigSearch}${zunitApplicationPgmSearch} # Rule to locate Cobol copy books. This rule defaults to the local copybook folder # in the main application folder. +# used in dependency resolution and impact analysis # ** deprecated ** copybookRule = {"library": "SYSLIB", \ "searchPath": [ \ @@ -112,13 +113,15 @@ copybookRule = {"library": "SYSLIB", \ # dependency resolution in the application directory and a shared common copybook location: # copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy;/u/build/common/copybooks/*.cpy # -# More samples can be found at: https://www.ibm.com/docs/en/adfz/dbb/1.1.0?topic=scripts-how-manage-build-dependencies +# More samples can be found along with the syntax for the search path configurations at: +# https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-manage-build-dependencies#6-resolving-logical-build-dependencies-to-local-physical-files # copybookSearch = search:${workspace}/?path=${application}/copybook/*.cpy # Rule to locate PLI include files. This rule defaults to the local plinc folder # in the main application folder. +# used in dependency resolution and impact analysis plincRule = {"library": "SYSLIB", \ "searchPath": [ \ {"sourceDir": "${workspace}", "directory": "${application}/plinc"} \ @@ -133,6 +136,7 @@ pliincludeSearch = search:${workspace}/?path=${application}/plinc/*.cpy # Rule to locate ASM macros. This rule defaults to the local maclib folder # in the main application folder. # Category filters on what you want to include during the scanner (i.e. excludes macro-def keyword) +# used in dependency resolution and impact analysis # ** deprecated ** maclibRule = {"library": "SYSLIB", "category": "MACRO", \ "searchPath": [ \ @@ -143,10 +147,11 @@ maclibRule = {"library": "SYSLIB", "category": "MACRO", \ # asmMacroSearch # searchPath to locate Assembler macro files # used in dependency resolution and impact analysis -asmMacroSearch = search:[SYSLIB:MARCO]${workspace}/?path=${application}/maclib/*.mac +asmMacroSearch = search:[SYSLIB:MACRO]${workspace}/?path=${application}/maclib/*.mac # Rule to locate ASM copybooks. This rule defaults to the local maclib folder # in the main application folder. +# used in dependency resolution and impact analysis # ** deprecated ** asmCopyRule = {"library": "SYSLIB", "category": "COPY", \ "searchPath": [ \ @@ -161,6 +166,7 @@ asmCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/maclib/*.m # # Rule to locate BMS maps +# used only in impact analysis # ** deprecated ** bmsRule = {"library": "SYSLIB", \ "searchPath": [ \ @@ -176,6 +182,7 @@ bmsSearch = search:${workspace}/?path=${application}/bms/*.bms # Rule to locate REXX includes. This rule defaults to the local rexx folder # in the main application folder. +# used in dependency resolution and impact analysis # ** deprecated ** rexxRule = {"library": "SYSLIB", "category": "COPY", \ "searchPath": [ \ @@ -189,21 +196,42 @@ rexxRule = {"library": "SYSLIB", "category": "COPY", \ # used in dependency resolution and impact analysis rexxCopySearch = search:[SYSLIB:COPY]${workspace}/?path=${application}/rexx/*.rexx -# Rule to locate Link files for rebuilding statically linked load modules - +# Rule to locate COBOL programs and Link files for rebuilding statically linked load modules +# searchPath configuration for dependencies to a COBOL program located in the cobol subfolder +# used only in impact analysis +# ** deprecated ** linkRule = {"category": "LINK", \ "searchPath": [ \ - {"sourceDir": "${workspace}", "directory": "${application}/link"} \ + {"sourceDir": "${workspace}", "directory": "${application}/cobol"} \ ] \ } # -# linkCardSearch -# searchPath to locate impacted linkcards +# linkSearch +# +# searchPath to locate impacted linkcards or main programs after an included submodule is changed +# leverages the output collection, which has the dependency info from the executable # category LINK only; used only in impact analysis -linkCardSearch = search:[:LINK]${workspace}/?path=${application}/link/*.lnk +# +# Additional samples: +# +# impact resolution across all directories in build workspace, but filtering on the file extension cbl: +# staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl +# +# impact resolution across all directories in build workspace, but filtering on the file extension cbl and pli (for cobol and pli submodules): +# staticLinkSearch = search:[:LINK]${workspace}/?path=**/*.cbl,**/*.pli +# +# More samples can be found along with the syntax for the search path configurations at: +# https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-manage-build-dependencies#6-resolving-logical-build-dependencies-to-local-physical-files +# +# Special case with Dependency Scanner Transfer Control Statement capturing turned on (default is off) +# the scanners detect a static call to the literal, which would need to turn into a new rule for CALL: +# staticCallSearch = search:[:CALL]${workspace}/?path=${application}/cobol/*.cbl +# +linkSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl # Rule to locate the zUnit test configuration file +# used in dependency resolution and impact analysis # ** deprecated ** -> zunitTestConfigSearch testconfigRule = {"library": "SYSPROG", \ "searchPath": [ \ @@ -217,6 +245,7 @@ testconfigRule = {"library": "SYSPROG", \ zunitTestConfigSearch = search:[SYSPROG:]${workspace}/?path=${application}/testcfg/*.bzucfg # Rule to locate the zUnit playback file +# used in dependency resolution and impact analysis # ** deprecated ** -> zunitPlayfileSearch testcaseRule = {"library": "SYSPLAY", \ "searchPath": [ \ @@ -232,6 +261,7 @@ zunitPlayfileSearch = search:[SYSPLAY:]${workspace}/?path=${application}/testpla # Rule to locate the zUnit test case program when the program or the bzucfg file changes +# used in impact analysis # ** deprecated ** -> zunitApplicationPgmSearch testcasePgmRule = {"category": "PROGRAMDEPENDENCY", \ "searchPath": [ \ @@ -244,7 +274,7 @@ testcasePgmRule = {"category": "PROGRAMDEPENDENCY", \ # searchPath to locate impacted application programs # see also build-conf/build.properties -> createTestcaseDependency # used in impact analysis -zunitApplicationPgmSearch = search:[:LINK]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli +zunitApplicationPgmSearch = search:[:PROGRAMDEPENDENCY]${workspace}/?path=${application}/cobol/*.cbl;${application}/pli/*.pli # Rule to detect DBB Build property changes # ** deprecated ** From d56aba8371c50d6a11ee5bc1c177825fa45629d7 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Thu, 2 Jun 2022 16:50:49 +0200 Subject: [PATCH 06/14] New CLI option to set or overwrite build property (#213) * implementing propOverwrite cli option * Update invocation sample --- BUILD.md | 162 ++++++++++++++++++++++++++++++++++++++++++++++++++- build.groovy | 20 ++++++- 2 files changed, 178 insertions(+), 4 deletions(-) diff --git a/BUILD.md b/BUILD.md index 7438506c..c8f75727 100644 --- a/BUILD.md +++ b/BUILD.md @@ -100,8 +100,7 @@ required options: -h,--hlq High level qualifier for partition data sets build options: - -p,--propFiles Comma separated list of additional property files - to load. Absolute paths or relative to workspace + -f,--fullBuild Flag indicating to build all programs for the application -i,--impactBuild Flag indicating to build only programs impacted @@ -124,7 +123,11 @@ build options: -l,--logEncoding Encoding of output logs. Default is EBCDIC directory for user build -zTest,--runzTests Specify if zUnit Tests should be run - + + -p,--propFiles Comma separated list of additional property files + to load. Absolute paths or relative to workspace + -po,--propOverwrites Comma separated list of key=value pairs for set and overwrite build properties + -cc,--ccczUnit Flag to indicate to collect code coverage reports during zUnit step -cch,--cccHost Headless Code Coverage Collector host (if not specified IDz will be used for reporting) -ccp,--cccPort Headless Code Coverage Collector port (if not specified IDz will be used for reporting) @@ -163,6 +166,7 @@ utility options - [Perform a Merge build](#perform-a-merge-build) - [Perform a Scan Source build](#perform-a-scan-source-build) - [Perform a Scan Source + Outputs build](#perform-a-scan-source--outputs-build) +- [Dynamically Overwrite build properties](#dynamically-overwrite-build-properties) ### Build a Single Program @@ -1415,3 +1419,155 @@ HTTP/1.1 200 OK ```
+ +### Dynamically Overwrite build properties + +Dependending on where are in your development process, you might want to dynamically overwrite a build property. One scenario is for example to set the `mainBuildBranch` option which is used also to clone collections and initialize the DBB collections. See [Perform Impact Build for topic branches](#perform-impact-build-for-topic-branches) + +To dynamically overwrite any build property, you can make use of the `--propOverwrite` cli argument. To overwrite a build property, pass it in as a key-value pair via the CLI. Several key-value pairs can be passed by comma-separating them :`--propOverwrite mainBuildBranch=develop,cobol_compilerVersion=V6`. Please note, that values which contain a comma (`,`) cannot be passed into the list. Please also make sure that you don't run into any limitations with regards of the length of the overall build command. For mass overwrites, use the `--propFiles` option to pass in one or several property files. + + +``` +groovyz dbb-zappbuild/build.groovy --workspace /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --impactBuild --verbose --propOverwrite mainBuildBranch=develop +``` +
+ Build log + +``` +** Build start at 20210622.082942.029 +** Input args = /var/dbb/dbb-zappbuild/samples --hlq DBB.ZAPP.CLEAN.MASTER --workDir /var/dbb/out/MortgageApplication --application MortgageApplication --logEncoding UTF-8 --impactBuild --verbose --propOverwrite mainBuildBranch=develop +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/datasets.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/Assembler.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/BMS.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/MFS.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/PSBgen.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/DBDgen.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/ACBgen.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/Cobol.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/LinkEdit.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/PLI.properties +** Loading property file /ZT01/var/dbb/dbb-zappbuild/build-conf/ZunitConfig.properties +** appConf = /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/file.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/BMS.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/Cobol.properties +** Loading property file /var/dbb/dbb-zappbuild/samples/MortgageApplication/application-conf/LinkEdit.properties +** Overwriting build property mainBuildBranch from cli argument --propOverwrite with value develop +..... // lists of all build properties +** Repository client created for https://dbb-webapp:8080/dbb +** Build output located at /var/dbb/out/MortgageApplication/build.20210622.082942.029 +** Build result created for BuildGroup:MortgageApplication-master BuildLabel:build.20210622.082942.029 at https://dbb-webapp:8080/dbb/rest/buildResult/47012 +** --impactBuild option selected. Building impacted programs for application MortgageApplication +** Getting current hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Storing MortgageApplication : 857266a44a6e859c4f949adb7e32cfbc4a8bd736 +** Getting baseline hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Storing MortgageApplication : 75e13783f2197e12772cec64a16937707ea623a5 +** Calculating changed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Diffing baseline 75e13783f2197e12772cec64a16937707ea623a5 -> current 857266a44a6e859c4f949adb7e32cfbc4a8bd736 +*** Changed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication: +**** MortgageApplication/copybook/epsmtcom.cpy +*** Deleted files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication: +*** Renamed files for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication: +** Updating collections MortgageApplication-master and MortgageApplication-master-outputs +*** Scanning file MortgageApplication/copybook/epsmtcom.cpy (/var/dbb/dbb-zappbuild/samples/MortgageApplication/copybook/epsmtcom.cpy) +*** Scanning file with the default scanner +*** Logical file for MortgageApplication/copybook/epsmtcom.cpy = +{"dli":false,"lname":"EPSMTCOM","file":"MortgageApplication\/copybook\/epsmtcom.cpy","mq":false,"cics":false,"logicalDependencies":[{"lname":"EPSMTINP","library":"SYSLIB","category":"COPY"},{"lname":"EPSMTOUT","library":"SYSLIB","category":"COPY"}],"language":"COB","sql":false} +** Storing 1 logical files in repository collection 'MortgageApplication-master' +HTTP/1.1 200 OK +** Performing impact analysis on changed file MortgageApplication/copybook/epsmtcom.cpy +*** Creating impact resolver for MortgageApplication/copybook/epsmtcom.cpy with [{"library": "SYSLIB", "searchPath": [ {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/copybook"} ] },{"library": "SYSLIB", "searchPath": [ {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/bms"} ] },{"category": "LINK", "searchPath": [ {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/cobol"}, {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/link"} ] }] rules +** Found impacted file MortgageApplication/cobol/epscsmrt.cbl +** MortgageApplication/cobol/epscsmrt.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/link/epsmlist.lnk +** MortgageApplication/link/epsmlist.lnk is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epscmort.cbl +** MortgageApplication/cobol/epscmort.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epsmlist.cbl +** MortgageApplication/cobol/epsmlist.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epscsmrt.cbl +** MortgageApplication/cobol/epscsmrt.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Found impacted file MortgageApplication/cobol/epscmort.cbl +** MortgageApplication/cobol/epscmort.cbl is impacted by changed file MortgageApplication/copybook/epsmtcom.cpy. Adding to build list. +** Writing build list file to /var/dbb/out/MortgageApplication/build.20210622.082942.029/buildList.txt +MortgageApplication/cobol/epsmlist.cbl +MortgageApplication/cobol/epscsmrt.cbl +MortgageApplication/cobol/epscmort.cbl +MortgageApplication/link/epsmlist.lnk +** Invoking build scripts according to build order: BMS.groovy,Cobol.groovy,LinkEdit.groovy +** Building files mapped to Cobol.groovy script +required props = cobol_srcPDS,cobol_cpyPDS,cobol_objPDS,cobol_loadPDS,cobol_compiler,cobol_linkEditor,cobol_tempOptions,applicationOutputsCollectionName, SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.COBOL +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.COPY +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.OBJ +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.DBRM +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.LOAD +*** Building file MortgageApplication/cobol/epsmlist.cbl +*** Creating dependency resolver for MortgageApplication/cobol/epsmlist.cbl with [{"library": "SYSLIB", "searchPath": [ {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/copybook"} ] }] rules +*** Scanning file with the default scanner +*** Resolution rules for MortgageApplication/cobol/epsmlist.cbl: +{"library":"SYSLIB","searchPath":[{"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","directory":"MortgageApplication\/copybook"}]} +*** Physical dependencies for MortgageApplication/cobol/epsmlist.cbl: +{"excluded":false,"lname":"DFHAID","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"lname":"EPSMLIS","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMORTF","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmortf.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTINP","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtinp.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTOUT","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtout.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTCOM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtcom.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSNBRPM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsnbrpm.cpy","category":"COPY","resolved":true} +Cobol compiler parms for MortgageApplication/cobol/epsmlist.cbl = LIB,CICS +*** Building file MortgageApplication/cobol/epscsmrt.cbl +*** Creating dependency resolver for MortgageApplication/cobol/epscsmrt.cbl with [{"library": "SYSLIB", "searchPath": [ {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/copybook"} ] }] rules +*** Scanning file with the default scanner +*** Resolution rules for MortgageApplication/cobol/epscsmrt.cbl: +{"library":"SYSLIB","searchPath":[{"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","directory":"MortgageApplication\/copybook"}]} +*** Physical dependencies for MortgageApplication/cobol/epscsmrt.cbl: +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTINP","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtinp.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTOUT","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtout.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTCOM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtcom.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSPDATA","library":"SYSLIB","file":"MortgageApplication\/copybook\/epspdata.cpy","category":"COPY","resolved":true} +Cobol compiler parms for MortgageApplication/cobol/epscsmrt.cbl = LIB,CICS +*** Scanning load module for MortgageApplication/cobol/epscsmrt.cbl +*** Logical file = +{"dli":false,"lname":"EPSCSMRT","file":"MortgageApplication\/cobol\/epscsmrt.cbl","mq":false,"cics":false,"language":"ZBND","sql":false} +*** Building file MortgageApplication/cobol/epscmort.cbl +*** Creating dependency resolver for MortgageApplication/cobol/epscmort.cbl with [{"library": "SYSLIB", "searchPath": [ {"sourceDir": "/var/dbb/dbb-zappbuild/samples", "directory": "MortgageApplication/copybook"} ] }] rules +*** Scanning file with the default scanner +*** Resolution rules for MortgageApplication/cobol/epscmort.cbl: +{"library":"SYSLIB","searchPath":[{"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","directory":"MortgageApplication\/copybook"}]} +*** Physical dependencies for MortgageApplication/cobol/epscmort.cbl: +{"excluded":false,"lname":"DFHAID","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"lname":"EPSMORT","library":"SYSLIB","category":"COPY","resolved":false} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTINP","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtinp.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTOUT","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtout.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSMTCOM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsmtcom.cpy","category":"COPY","resolved":true} +{"excluded":false,"sourceDir":"\/var\/dbb\/dbb-zappbuild\/samples","lname":"EPSNBRPM","library":"SYSLIB","file":"MortgageApplication\/copybook\/epsnbrpm.cpy","category":"COPY","resolved":true} +{"excluded":false,"lname":"SQLCA","library":"SYSLIB","category":"SQL INCLUDE","resolved":false} +Cobol compiler parms for MortgageApplication/cobol/epscmort.cbl = LIB,CICS,SQL +*** Scanning load module for MortgageApplication/cobol/epscmort.cbl +*** Logical file = +{"dli":false,"lname":"EPSCMORT","file":"MortgageApplication\/cobol\/epscmort.cbl","mq":false,"cics":false,"logicalDependencies":[{"lname":"EPSNBRVL","library":"DBB.ZAPP.CLEAN.MASTER.OBJ","category":"LINK"}],"language":"ZBND","sql":false} +** Building files mapped to LinkEdit.groovy script +required props = linkedit_srcPDS,linkedit_objPDS,linkedit_loadPDS,linkedit_linkEditor,linkedit_tempOptions,applicationOutputsCollectionName, SDFHLOAD,SCEELKED +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.LINK +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.OBJ +** Creating / verifying build dataset DBB.ZAPP.CLEAN.MASTER.LOAD +*** Building file MortgageApplication/link/epsmlist.lnk +*** Creating dependency resolver for MortgageApplication/link/epsmlist.lnk with null rules +*** Scanning file with the default scanner +*** Scanning load module for MortgageApplication/link/epsmlist.lnk +*** Logical file = +{"dli":false,"lname":"EPSMLIST","file":"MortgageApplication\/link\/epsmlist.lnk","mq":false,"cics":false,"logicalDependencies":[{"lname":"EPSMPMT","library":"DBB.ZAPP.CLEAN.MASTER.LOAD","category":"LINK"},{"lname":"EPSMLIST","library":"DBB.ZAPP.CLEAN.MASTER.OBJ","category":"LINK"}],"language":"ZBND","sql":false} +*** Obtaining hash for directory /var/dbb/dbb-zappbuild/samples/MortgageApplication +** Setting property :githash:MortgageApplication : 857266a44a6e859c4f949adb7e32cfbc4a8bd736 +** Setting property :giturl:MortgageApplication : git@github.ibm.com:zDevOps-Acceleration/dbb-zappbuild.git +** Writing build report data to /var/dbb/out/MortgageApplication/build.20210622.082942.029/BuildReport.json +** Writing build report to /var/dbb/out/MortgageApplication/build.20210622.082942.029/BuildReport.html +** Updating build result BuildGroup:MortgageApplication-master BuildLabel:build.20210622.082942.029 at https://dbb-webapp:8080/dbb/rest/buildResult/47012 +** Build ended at Tue Jun 22 08:29:59 GMT+01:00 2021 +** Build State : CLEAN +** Total files processed : 4 +** Total build time : 17.239 seconds +``` + +
diff --git a/build.groovy b/build.groovy index c4bd9146..9498cc81 100644 --- a/build.groovy +++ b/build.groovy @@ -184,7 +184,8 @@ options: cli.h(longOpt:'hlq', args:1, required:true, 'High level qualifier for partition data sets') // build options - cli.p(longOpt:'propFiles', args:1, 'Commas spearated list of additional property files to load. Absolute paths or relative to workspace.') + cli.p(longOpt:'propFiles', args:1, 'Comma separated list of additional property files to load. Absolute paths or relative to workspace.') + cli.po(longOpt:'propOverwrites', args:1, 'Comma separated list of key=value pairs for set and overwrite build properties.') cli.l(longOpt:'logEncoding', args:1, 'Encoding of output logs. Default is EBCDIC') cli.f(longOpt:'fullBuild', 'Flag indicating to build all programs for application') cli.i(longOpt:'impactBuild', 'Flag indicating to build only programs impacted by changed files since last successful build.') @@ -326,6 +327,23 @@ def populateBuildProperties(String[] args) { props.load(new File(propFile)) } } + + // populate property overwrites from argument list + if (opts.po) props.propOverwrites = opts.po + if (props.propOverwrites) { + String[] propOverwrites = props.propOverwrites.split(',') + propOverwrites.each { buildPropertyOverwrite -> + (key, value) = buildPropertyOverwrite.tokenize('=') + if (key && value) { + if (opts.v) println "** Overwriting build property ${key} from cli argument --propOverwrite with value ${value}" + props.put(key, value) + } + else { + println "*! Overwriting build property from cli argument --propOverwrite failed due a null value ( key: $key , value :$value )" + } + } + } + // set flag indicating to run unit tests if (opts.zTest) props.runzTests = 'true' From 6e4269c758d4950626708f00f62102ebff749de2 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 7 Jun 2022 16:55:16 +0200 Subject: [PATCH 07/14] Update zUnit Config script to support Groovy 4.0 runtime (#219) --- languages/ZunitConfig.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/languages/ZunitConfig.groovy b/languages/ZunitConfig.groovy index d8756412..3b21a0fe 100644 --- a/languages/ZunitConfig.groovy +++ b/languages/ZunitConfig.groovy @@ -3,6 +3,7 @@ import com.ibm.dbb.repository.* import com.ibm.dbb.dependency.* import com.ibm.dbb.build.* import groovy.transform.* +import groovy.xml.* // define script properties From 50cec081e2dcc3234689afb5072e2e673a24ae0c Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Wed, 8 Jun 2022 16:19:42 +0200 Subject: [PATCH 08/14] Mark build as failed on git errors (#203) * flag build as in error when git diff command fails #202 --- utilities/GitUtilities.groovy | 54 ++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/utilities/GitUtilities.groovy b/utilities/GitUtilities.groovy index 22598a4e..d18b3e4a 100644 --- a/utilities/GitUtilities.groovy +++ b/utilities/GitUtilities.groovy @@ -4,6 +4,9 @@ import com.ibm.dbb.dependency.* import com.ibm.dbb.build.* import groovy.transform.* +@Field BuildProperties props = BuildProperties.getInstance() +@Field RepositoryClient repositoryClient + /* * Tests if directory is in a local git repository * @@ -20,7 +23,9 @@ def isGitDir(String dir) { Process process = cmd.execute() process.waitForProcessOutput(gitResponse, gitError) if (gitError) { - println("*? Warning executing isGitDir($dir). Git command: $cmd error: $gitError") + String warningMsg = "*? Warning executing isGitDir($dir). Git command: $cmd error: $gitError" + println(warningMsg) + updateBuildResult(warningMsg:warningMsg,client:getRepositoryClient()) } else if (gitResponse) { isGit = gitResponse.toString().trim().toBoolean() @@ -137,7 +142,10 @@ def getCurrentGitHash(String gitDir, boolean abbrev) { Process process = cmd.execute() process.waitForProcessOutput(gitHash, gitError) if (gitError) { - print("*! Error executing Git command: $cmd error: $gitError") + String errorMsg = "*! Error executing Git command: $cmd error: $gitError" + println(errorMsg) + props.error = "true" + updateBuildResult(errorMsg:errorMsg,client:getRepositoryClient()) } return gitHash.toString().trim() } @@ -249,8 +257,10 @@ def getChangedFiles(String cmd) { // handle command error if (git_error.size() > 0) { - println("*! Error executing Git command: $cmd error: $git_error") - println ("*! Attempting to parse unstable git command for changed files...") + String errorMsg = "*! Error executing Git command: $cmd error: $git_error \n *! Attempting to parse unstable git command for changed files..." + println(errorMsg) + props.error = "true" + updateBuildResult(errorMsg:errorMsg,client:getRepositoryClient()) } for (line in git_diff.toString().split("\n")) { @@ -369,4 +379,40 @@ def getChangedProperties(String gitDir, String baseline, String currentHash, Str } return changedProperties.propertyNames() +} + +/** helper methods **/ + +def getRepositoryClient() { + if (!repositoryClient && props."dbb.RepositoryClient.url") + repositoryClient = new RepositoryClient().forceSSLTrusted(true) + return repositoryClient +} + +/* + * updateBuildResult - for git cmd related issues + */ +def updateBuildResult(Map args) { + // args : errorMsg / warningMsg, client:repoClient + + // update build results only in non-userbuild scenarios + if (args.client && !props.userBuild) { + def buildResult = args.client.getBuildResult(props.applicationBuildGroup, props.applicationBuildLabel) + if (!buildResult) { + println "*! No build result found for BuildGroup '${props.applicationBuildGroup}' and BuildLabel '${props.applicationBuildLabel}'" + return + } + // add error message + if (args.errorMsg) { + buildResult.setStatus(buildResult.ERROR) + buildResult.addProperty("error", args.errorMsg) + } + // add warning message, but keep result status + if (args.warningMsg) { + // buildResult.setStatus(buildResult.WARNING) + buildResult.addProperty("warning", args.warningMsg) + } + // save result + buildResult.save() + } } \ No newline at end of file From 24e49abf0001d4ca3584d9a26c459f44eb092d28 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Fri, 10 Jun 2022 17:50:45 +0200 Subject: [PATCH 09/14] Replace usage of evaluate with string parsing (#217) * replacing the evaluate methods by parsing a JSON String --- build-conf/Cobol.properties | 4 ++-- build-conf/PLI.properties | 4 ++-- languages/Cobol.groovy | 16 ++++++++++++---- languages/PLI.groovy | 16 ++++++++++++---- utilities/BuildUtilities.groovy | 21 ++++++++++++++++++++- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/build-conf/Cobol.properties b/build-conf/Cobol.properties index 6a2055b0..b063330f 100644 --- a/build-conf/Cobol.properties +++ b/build-conf/Cobol.properties @@ -83,13 +83,13 @@ dbb.LinkEditScanner.excludeFilter = ${SDFHLOAD}.*, ${SCEELKED}.* # additional libraries for linkEdit SYSLIB concatenation, comma-separated, see definitions in application-conf # cobol_linkEditSyslibConcatenation= -# cobol_dependenciesAlternativeLibraryNameMapping - an *optional* map to define target dataset definition for alternate include libraries +# cobol_dependenciesAlternativeLibraryNameMapping - an *optional* JSON String of a map defining the target dataset definition for alternate include libraries # this property is used to # * copy files the to mapped dataset definition (PLEASE NOTE! This setting takes precendence over cobol_dependenciesDatasetMapping) # * defining additional allocations in the compile step # # note that the SYSLIB is defaulted to the dataset definition 'cobol_cpyPDS' and is not required to be set here -# sample: cobol_dependenciesAlternativeLibraryNameMapping = [MYFILE: 'cobol_myfilePDS', DCLGEN : 'cobol_dclgenPDS'] +# sample: cobol_dependenciesAlternativeLibraryNameMapping = {"MYFILE": "cobol_myfilePDS", "DCLGEN" : "cobol_dclgenPDS"} cobol_dependenciesAlternativeLibraryNameMapping= # cobol_dependenciesDatasetMapping - an optional dbb property mapping to map dependencies to different target datasets diff --git a/build-conf/PLI.properties b/build-conf/PLI.properties index cf837439..60c3764d 100644 --- a/build-conf/PLI.properties +++ b/build-conf/PLI.properties @@ -70,12 +70,12 @@ dbb.DependencyScanner.languageHint=PLI :: **/*.pli, **/*.inc, **/*.cpy # additional libraries for linkEdit SYSLIB concatenation, comma-separated, see definitions in application-conf # pli_linkEditSyslibConcatenation= -# pli_dependenciesAlternativeLibraryNameMapping - an *optional* map to define target dataset definition for alternate include libraries +# pli_dependenciesAlternativeLibraryNameMapping - an *optional* JSON String of a map defining the target dataset definition for alternate include libraries # this property is used to # * copy files the to mapped dataset definition (this setting takes precendence over pli_dependenciesDatasetMapping) # * defining additional allocations in the compile step # note that the SYSLIB is defaulted to the dataset definition 'pli_cpyPDS' and is not required to be set here -# sample: pli_dependenciesAlternativeLibraryNameMapping = [MYFILE: 'pli_myfilePDS', DCLGEN : 'pli_dclgenPDS'] +# sample: pli_dependenciesAlternativeLibraryNameMapping = {"MYFILE": "pli_myfilePDS", "DCLGEN" : "pli_dclgenPDS"} pli_dependenciesAlternativeLibraryNameMapping= # pli_dependenciesDatasetMapping - an optional dbb property mapping to map dependencies to different target datasets diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 55c56792..44e18710 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -255,10 +255,18 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe // adding alternate library definitions if (props.cobol_dependenciesAlternativeLibraryNameMapping) { - alternateLibraryNameAllocations = evaluate(props.cobol_dependenciesAlternativeLibraryNameMapping) - alternateLibraryNameAllocations.each { libraryName, datasetDSN -> - datasetDSN = props.getProperty(datasetDSN) - if (datasetDSN) compile.dd(new DDStatement().name(libraryName).dsn(datasetDSN).options("shr")) + alternateLibraryNameAllocations = buildUtils.parseJSONStringToMap(props.cobol_dependenciesAlternativeLibraryNameMapping) + alternateLibraryNameAllocations.each { libraryName, datasetDefinition -> + datasetName = props.getProperty(datasetDefinition) + if (datasetName) { + compile.dd(new DDStatement().name(libraryName).dsn(datasetName).options("shr")) + } + else { + String errorMsg = "*! Cobol.groovy. The dataset definition $datasetDefinition could not be resolved from the DBB Build properties." + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg,client:getRepositoryClient()) + } } } diff --git a/languages/PLI.groovy b/languages/PLI.groovy index aee55b9c..553a7e58 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -239,10 +239,18 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe // adding alternate library definitions if (props.cobol_dependenciesAlternativeLibraryNameMapping) { - alternateLibraryNameAllocations = evaluate(props.pli_dependenciesAlternativeLibraryNameMapping) - alternateLibraryNameAllocations.each { libraryName, datasetDSN -> - datasetDSN = props.getProperty(datasetDSN) - if (datasetDSN) compile.dd(new DDStatement().name(libraryName).dsn(datasetDSN).options("shr")) + alternateLibraryNameAllocations = buildUtils.parseJSONStringToMap(props.pli_dependenciesAlternativeLibraryNameMapping) + alternateLibraryNameAllocations.each { libraryName, datasetDefinition -> + datasetName = props.getProperty(datasetDefinition) + if (datasetName) { + compile.dd(new DDStatement().name(libraryName).dsn(datasetName).options("shr")) + } + else { + String errorMsg = "*! PLI.groovy. The dataset definition $datasetDefinition could not be resolved from the DBB Build properties." + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg,client:getRepositoryClient()) + } } } diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 27fc6b8f..f6abe971 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -196,7 +196,7 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap // 2. langprefix_dependenciesDatasetMapping as a manual overwrite to determine an alternative library used in the default dd concatentation String dependencyPDS if (!physicalDependency.getLibrary().equals("SYSLIB") && dependenciesAlternativeLibraryNameMapping) { - dependencyPDS = props.getProperty(evaluate(dependenciesAlternativeLibraryNameMapping).get(physicalDependency.getLibrary())) + dependencyPDS = props.getProperty(parseJSONStringToMap(dependenciesAlternativeLibraryNameMapping).get(physicalDependency.getLibrary())) } if (dependencyPDS == null && dependenciesDatasetMapping){ dependencyPDS = props.getProperty(dependenciesDatasetMapping.getValue(physicalDependency.getFile())) @@ -742,6 +742,25 @@ def printPhysicalDependencies(List physicalDependencies) { } } +/* + * This is a helper method which parses a JSON String representing a map of key value pairs to a proper map + * e.q. cobol_dependenciesAlternativeLibraryNameMapping = {"MYFILE": "cobol_myfilePDS", "DCLGEN" : "cobol_dclgenPDS"} + */ + +def parseJSONStringToMap(String buildProperty) { + Map map = [:] + try { + JsonSlurper slurper = new groovy.json.JsonSlurper() + map = slurper.parseText(buildProperty) + } catch (Exception e) { + String errorMsg = "*! BuildUtils.parseJSONStringToMap - Failed to parse build property $buildProperty from JSON String into a map object. Marking build as in error." + println(errorMsg) + props.error = "true" + updateBuildResult(errorMsg:errorMsg) + } + return map +} + /* * Obtain the abbreviated git hash from the PropertyMappings table * returns null if no hash was found From 1875e8da815bb04d4daca2be9ea863f19bc9b6c3 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 13 Jun 2022 15:56:39 +0200 Subject: [PATCH 10/14] Correction of PSBGen script on obtaining the deployType for acbgen (#226) resolves #222 --- languages/PSBgen.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/languages/PSBgen.groovy b/languages/PSBgen.groovy index e68fbd09..1e783e2d 100644 --- a/languages/PSBgen.groovy +++ b/languages/PSBgen.groovy @@ -221,7 +221,7 @@ def createACBgenCommand(String buildFile, String member, File logFile) { // retrieve target pds and deploytype String acbgen_loadPDS = props.getFileProperty('acbgen_loadPDS', buildFile) - String deployType = buildUtils.getDeployType("acbgen", buildFile, logicalFile) + String deployType = buildUtils.getDeployType("acbgen", buildFile, null) acbgen.dd(new DDStatement().name("IMSACB").dsn("${acbgen_loadPDS}").options('shr').output(true).deployType(deployType)) // addional allocations From 6db188e938c9185818779d2f8987df60389329f6 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 13 Jun 2022 15:57:47 +0200 Subject: [PATCH 11/14] Correction of PLI script to correctly check the PLI alternate library configuration (#224) resolves #221 --- languages/PLI.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/languages/PLI.groovy b/languages/PLI.groovy index 553a7e58..bce56dc7 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -203,7 +203,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe // add additional datasets with dependencies based on the dependenciesDatasetMapping PropertyMappings dsMapping = new PropertyMappings('pli_dependenciesDatasetMapping') dsMapping.getValues().each { targetDataset -> - // exclude the defaults cobol_cpyPDS and any overwrite in the alternativeLibraryNameMap + // exclude the defaults pli_cpyPDS and any overwrite in the alternativeLibraryNameMap if (targetDataset != 'pli_incPDS') compile.dd(new DDStatement().dsn(props.getProperty(targetDataset)).options("shr")) } @@ -238,7 +238,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe compile.dd(new DDStatement().name("DBRMLIB").dsn("$props.pli_dbrmPDS($member)").options('shr').output(true).deployType('DBRM')) // adding alternate library definitions - if (props.cobol_dependenciesAlternativeLibraryNameMapping) { + if (props.pli_dependenciesAlternativeLibraryNameMapping) { alternateLibraryNameAllocations = buildUtils.parseJSONStringToMap(props.pli_dependenciesAlternativeLibraryNameMapping) alternateLibraryNameAllocations.each { libraryName, datasetDefinition -> datasetName = props.getProperty(datasetDefinition) @@ -258,7 +258,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe if (props.errPrefix) { compile.dd(new DDStatement().name("SYSADATA").options("DUMMY")) // SYSXMLSD.XML suffix is mandatory for IDZ/ZOD to populate remote error list - compile.dd(new DDStatement().name("SYSXMLSD").dsn("${props.hlq}.${props.errPrefix}.SYSXMLSD.XML").options(props.cobol_compileErrorFeedbackXmlOptions)) + compile.dd(new DDStatement().name("SYSXMLSD").dsn("${props.hlq}.${props.errPrefix}.SYSXMLSD.XML").options(props.pli_compileErrorFeedbackXmlOptions)) } // add a copy command to the compile command to copy the SYSPRINT from the temporary dataset to an HFS log file From 06d81c57c9b70cb9bb16b09cadc97d6ca5e4b010 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Mon, 13 Jun 2022 15:59:12 +0200 Subject: [PATCH 12/14] Removal of Error Feedback in REXX.groovy while not available for REXX (#225) --- languages/REXX.groovy | 7 ------- 1 file changed, 7 deletions(-) diff --git a/languages/REXX.groovy b/languages/REXX.groovy index 0a4b5fd7..0c7a2289 100644 --- a/languages/REXX.groovy +++ b/languages/REXX.groovy @@ -163,13 +163,6 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe if (props.SFELLOAD) compile.dd(new DDStatement().dsn(props.SFELLOAD).options("shr")) - // add IDz User Build Error Feedback DDs - if (props.errPrefix) { - compile.dd(new DDStatement().name("SYSADATA").options("DUMMY")) - // SYSXMLSD.XML suffix is mandatory for IDZ/ZOD to populate remote error list - compile.dd(new DDStatement().name("SYSXMLSD").dsn("${props.hlq}.${props.errPrefix}.SYSXMLSD.XML").options(props.cobol_compileErrorFeedbackXmlOptions)) - } - // add a copy command to the compile command to copy the SYSPRINT from the temporary dataset to an HFS log file compile.copy(new CopyToHFS().ddName("SYSPRINT").file(logFile).hfsEncoding(props.logEncoding)) From 2186911df2fd09e72c1e511e2ac7af791d134cd9 Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 14 Jun 2022 07:35:47 +0200 Subject: [PATCH 13/14] Ability to manage member level build properties in specific prop files (#216) * Ability to manage member level build properties in specific property files * Documentation of the build property management section in samples/application-conf/application.properties Co-authored-by: Lauren Li <45975633+lauren-li@users.noreply.github.com> --- build.groovy | 4 ++ .../application-conf/application.properties | 64 +++++++++++++++++++ .../application-conf/file.properties | 10 ++- .../properties/epsmlist.cbl.properties | 12 ++++ samples/application-conf/README.md | 7 +- .../application-conf/application.properties | 63 ++++++++++++++++++ utilities/BuildUtilities.groovy | 34 ++++++++++ 7 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 samples/MortgageApplication/properties/epsmlist.cbl.properties diff --git a/build.groovy b/build.groovy index 9498cc81..2681b4e4 100644 --- a/build.groovy +++ b/build.groovy @@ -552,6 +552,10 @@ def createBuildList() { println "** Scanning source code." impactUtils.updateCollection(buildList, null, null, repositoryClient) } + + // Loading file/member level properties from member specific properties files + println "** Populate file level properties from individual property files." + buildUtils.loadFileLevelPropertiesFromFile(buildList) return [buildList, deleteList] } diff --git a/samples/MortgageApplication/application-conf/application.properties b/samples/MortgageApplication/application-conf/application.properties index 085dd17a..b7271161 100644 --- a/samples/MortgageApplication/application-conf/application.properties +++ b/samples/MortgageApplication/application-conf/application.properties @@ -35,14 +35,78 @@ excludeFileList=.*,**/.*,**/*.properties,**/*.xml,**/*.groovy,**/*.json,**/*.md, # sample: skipImpactCalculationList=**/epsmtout.cpy,**/centralCopybooks/*.cpy skipImpactCalculationList= + +############################################################### +# Build Property management +############################################################### +# +# zAppBuild allows you to manage default properties and file properties: +# +# - Default build properties are defined in the /build-conf and /application-conf property files (e.g. Cobol.properties) +# +# - File properties override corresponding default build properties, and are defined through one of two methods: +# - Overwrites for groups or individual files +# - Typically defined in file.properties using the DBB file property path syntax +# - See: https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-organize-build-script#file-properties +# - Overwrites for an individual file +# - Defined in an individual property file located in a configurable subfolder (e.g. properties/epsmlist.cbl.properties) +# +# A typical scenario for using zAppBuild's capability to set build properties for an individual source file via a corresponding +# individual property file is to overwrite compile or link options. This approach might help ease the migration of properties +# from the previous build system. +# +# Individual property files are resolved using the pattern /.. For example, +# for the source file epsmlist.cbl, the process searches for a file in the propertyFilePath directory +# with the name epsmlist.cbl.properties. +# If no corresponding property file is found, the build will use the default build values. (Or, if any file properties were defined +# using the DBB file property path syntax, then the build will use those.) +# +# Note: Overwrites for a specific build property should be managed either via the file property path syntax or +# in the individual property files, but not both. The following example shows how both approaches for defining +# file properties can be combined to specify a set of build properties for the same source file: +# +# ### Example: Using the file property path syntax and individual property files to define build properties for a source file named app/cobol/AB123456.cbl +# - You can use the file property path syntax to define a file property for a group of files. The below defines the deployType for +# all source files in the folder cobol beginning with AB* to be BATCHLOAD: +# +# cobol_deployType = BATCHLOAD :: **/cobol/AB*.cbl +# +# - At the same time, you can define an individual file property file for app/cobol/AB123456.cbl with the following build property: +# +# cobol_compileParms = LIB,SOURCE +# +# - During the build, the file app/cobol/AB123456.cbl will have the deployType BATCHLOAD and the COBOL compile parameters LIB and SOURCE. +# ### End example ### + +# ### Properties to enable and configure build property overwrites using individual property files + +# flag to enable the zAppBuild capability to load individual property files for individual source files, +# Note: To activate the loading of property files for a group of files, it is recommended to use the file property path +# syntax instead. (i.e. loadFileLevelProperties=true :: **/cobol/*.cbl to enable it for all cobol files) +# default: false +loadFileLevelProperties=false + +# relative path to folder containing individual property files +# assumed to be relative to ${workspace}/${application} +propertyFilePath=properties + +# file extension for individual property files +# default: properties +propertyFileExtension=properties + +############################################################### +# Dependency Analysis and Impact Analysis configuration +############################################################### # # flag to define which DBB API is used for dependency and impact analysis # # false = uses the DependencyResolver and ImpactResolver APIs (default) # please note that DependencyResolver and ImpactResolver APIs are deprecated # configuration is based on the lang_resolutionRules and impactResolutionRules build properties +# # true = leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 # configuration is based on the lang_dependencySearch and impactSearch build properties +# useSearchConfiguration=false # diff --git a/samples/MortgageApplication/application-conf/file.properties b/samples/MortgageApplication/application-conf/file.properties index ef754875..cc65f78d 100644 --- a/samples/MortgageApplication/application-conf/file.properties +++ b/samples/MortgageApplication/application-conf/file.properties @@ -19,4 +19,12 @@ cobol_linkEdit = false :: **/cobol/epsnbrvl.cbl, **/cobol/epsmlist.cbl # # epsmlist needs to compile as CICS but does not have EXEC CICS statements # so is not automatically flagged as CICS by dependency scanner -isCICS = true :: **/cobol/epsmlist.cbl, **/link/epsmlist.lnk, **/cobol/epscsmrt.cbl \ No newline at end of file +isCICS = true :: **/cobol/epsmlist.cbl, **/link/epsmlist.lnk, **/cobol/epscsmrt.cbl + +# +# Property File Management +# +# Flag to enable the zAppBuild capability to load individual property files for source files, +# This file path pattern looks for an individual property file for epsmlist.cbl. +# Please see detailed description in application.properties +loadFileLevelProperties = true :: **/cobol/epsmlist.cbl \ No newline at end of file diff --git a/samples/MortgageApplication/properties/epsmlist.cbl.properties b/samples/MortgageApplication/properties/epsmlist.cbl.properties new file mode 100644 index 00000000..a2809069 --- /dev/null +++ b/samples/MortgageApplication/properties/epsmlist.cbl.properties @@ -0,0 +1,12 @@ +# Sample of an individual property files overwrite for file cobol/epsmlist.cbl + +# For details, check out section in application.properties + +# +# Sample to merge properties of this file level overwrite with the default setting for cobol_compileParms +# A more exclusive overwrite would be cobol_compileParms=SOURCE +cobol_compileParms=${cobol_compileParms},SOURCE + +# +# Defines exclusively the compile options for this file +#cobol_compilerVersion=V4 \ No newline at end of file diff --git a/samples/application-conf/README.md b/samples/application-conf/README.md index cd39a837..9cf708a8 100644 --- a/samples/application-conf/README.md +++ b/samples/application-conf/README.md @@ -23,13 +23,18 @@ gitRepositoryURL | git repository URL of the application repository to establis excludeFileList | Files to exclude when scanning or running full build. | false skipImpactCalculationList | Files for which the impact analysis should be skipped in impact build | false jobCard | JOBCARD for JCL execs | false +**Build Property management** | | +loadFileLevelProperties | Flag to enable the zAppBuild capability to load individual property files for a build file | true +propertyFilePath | relative path to folder containing individual property files | true +propertyFileExtension | file extension for individual property files | true +**Dependency and Impact resolution configuration** || useSearchConfiguration | Flag to define which DBB API is used for dependency and impact analysis. `false` uses DependencyResolver and ImpactResolver APIs, while `true` leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 | false resolveSubsystems | boolean flag to configure the SearchPathDependencyResolver to evaluate if resolved dependencies impact the file flags isCICS, isSQL, isDLI, isMQ when creating the LogicalFile | false impactResolutionRules | Comma separated list of resolution rule properties used for impact builds. Sample resolution rule properties (in JSON format) are included below. ** deprecated ** Please consider moving to new SearchPathDepedencyAPI leveraging `impactSearch` configuration. | true, recommended in file.properties impactSearch | Impact finder resolution search configuration leveraging the SearchPathImpactFinder API. Sample configurations are inlcuded below, next to the previous rule definitions. | true ### file.properties -Location of file properties, script mappings and file level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for column 'Overridable' in the tables below for build properties that can have file level property overrides. +Location of file properties, script mappings and file-level property overrides. All file properties for the entire application, including source files in distributed repositories of the application need to be contained either in this file or in other property files in the `application-conf` directory. Look for the column 'Overridable' in the tables below for build properties that can have file-level property overrides. Additional file-level properties can be defined through individual property files in a separate directory of the repository. For more details, see the section _Build Property Management_ in `application.properties` Property | Description --- | --- diff --git a/samples/application-conf/application.properties b/samples/application-conf/application.properties index 912b15d8..1fdb7876 100644 --- a/samples/application-conf/application.properties +++ b/samples/application-conf/application.properties @@ -55,14 +55,77 @@ skipImpactCalculationList= # Example: jobCard=//RUNZUNIT JOB ,MSGCLASS=H,CLASS=A,NOTIFY=&SYSUID,REGION=0M jobCard= +############################################################### +# Build Property management +############################################################### +# +# zAppBuild allows you to manage default properties and file properties: +# +# - Default build properties are defined in the /build-conf and /application-conf property files (e.g. Cobol.properties) +# +# - File properties override corresponding default build properties, and are defined through one of two methods: +# - Overwrites for groups or individual files +# - Typically defined in file.properties using the DBB file property path syntax +# - See: https://www.ibm.com/docs/en/dbb/1.1.0?topic=scripts-how-organize-build-script#file-properties +# - Overwrites for an individual file +# - Defined in an individual property file located in a configurable subfolder (e.g. properties/epsmlist.cbl.properties) +# +# A typical scenario for using zAppBuild's capability to set build properties for an individual source file via a corresponding +# individual property file is to overwrite compile or link options. This approach might help ease the migration of properties +# from the previous build system. +# +# Individual property files are resolved using the pattern /.. For example, +# for the source file epsmlist.cbl, the process searches for a file in the propertyFilePath directory +# with the name epsmlist.cbl.properties. +# If no corresponding property file is found, the build will use the default build values. (Or, if any file properties were defined +# using the DBB file property path syntax, then the build will use those.) +# +# Note: Overwrites for a specific build property should be managed either via the file property path syntax or +# in the individual property files, but not both. The following example shows how both approaches for defining +# file properties can be combined to specify a set of build properties for the same source file: +# +# ### Example: Using the file property path syntax and individual property files to define build properties for a source file named app/cobol/AB123456.cbl +# - You can use the file property path syntax to define a file property for a group of files. The below defines the deployType for +# all source files in the folder cobol beginning with AB* to be BATCHLOAD: +# +# cobol_deployType = BATCHLOAD :: **/cobol/AB*.cbl +# +# - At the same time, you can define an individual file property file for app/cobol/AB123456.cbl with the following build property: +# +# cobol_compileParms = LIB,SOURCE +# +# - During the build, the file app/cobol/AB123456.cbl will have the deployType BATCHLOAD and the COBOL compile parameters LIB and SOURCE. +# ### End example ### + +# ### Properties to enable and configure build property overwrites using individual property files + +# flag to enable the zAppBuild capability to load individual property files for individual source files, +# Note: To activate the loading of property files for a group of files, it is recommended to use the file property path +# syntax instead. (i.e. loadFileLevelProperties=true :: **/cobol/*.cbl to enable it for all cobol files) +# default: false +loadFileLevelProperties=false + +# relative path to folder containing individual property files +# assumed to be relative to ${workspace}/${application} +propertyFilePath=properties + +# file extension for individual property files +# default: properties +propertyFileExtension=properties + +############################################################### +# Dependency Analysis and Impact Analysis configuration +############################################################### # # flag to define which DBB API is used for dependency and impact analysis # # false = uses the DependencyResolver and ImpactResolver APIs (default) # please note that DependencyResolver and ImpactResolver APIs are deprecated # configuration is based on the lang_resolutionRules and impactResolutionRules build properties +# # true = leverages the DBB SearchPathDependencyResolver and SearchParthImpactFinder APIs introduced with DBB 1.1.2 # configuration is based on the lang_dependencySearch and impactSearch build properties +# useSearchConfiguration=false # diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index f6abe971..7ae57aab 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -773,3 +773,37 @@ def getShortGitHash(String buildFile) { if (props.verbose) println "*! Could not obtain abbreviated githash for buildFile $buildFile" return null } + +/* + * Loading file level properties for all files on the buildList or list which is passed to this method. + */ +def loadFileLevelPropertiesFromFile(List buildList) { + + buildList.each { String buildFile -> + + // check for file level overwrite + if (props.getFileProperty('loadFileLevelProperties', buildFile).toBoolean()) { + + String member = new File(buildFile).getName() + String propertyFilePath = props.getFileProperty('propertyFilePath', buildFile) + String propertyExtention = props.getFileProperty('propertyFileExtension', buildFile) + String propertyFile = getAbsolutePath(props.application) + "/${propertyFilePath}/${member}.${propertyExtention}" + File fileLevelPropFile = new File(propertyFile) + + if (fileLevelPropFile.exists()) { + if (props.verbose) println "* Populating property file $propertyFile for $buildFile" + InputStream propertyFileIS = new FileInputStream(propertyFile) + Properties fileLevelProps = new Properties() + fileLevelProps.load(propertyFileIS) + + fileLevelProps.entrySet().each { entry -> + if (props.verbose) println "* Adding file level pattern $entry.key = $entry.value for $buildFile" + props.addFilePattern(entry.key, entry.value, buildFile) + } + } else { + if (props.verbose) println "* No property file found for $buildFile. Build will take the defaults or already defined file properties." + } + } + } +} + From a4e1b993459433827f4d04f7ccd77e711917177a Mon Sep 17 00:00:00 2001 From: Dennis Behm Date: Tue, 14 Jun 2022 13:48:12 +0200 Subject: [PATCH 14/14] Correction when loading individual file properties (#228) * Update loadFileLevelProperties utility method to address scenario when loadFileLevelProperties build properties is not set in application-conf --- build.groovy | 6 ++++-- utilities/BuildUtilities.groovy | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/build.groovy b/build.groovy index 2681b4e4..54638f85 100644 --- a/build.groovy +++ b/build.groovy @@ -554,8 +554,10 @@ def createBuildList() { } // Loading file/member level properties from member specific properties files - println "** Populate file level properties from individual property files." - buildUtils.loadFileLevelPropertiesFromFile(buildList) + if (props.filePropertyValueKeySet().getAt("loadFileLevelProperties") || props.loadFileLevelProperties) { + println "** Populating file level properties from individual property files." + buildUtils.loadFileLevelPropertiesFromFile(buildList) + } return [buildList, deleteList] } diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 7ae57aab..db1deb7d 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -782,7 +782,8 @@ def loadFileLevelPropertiesFromFile(List buildList) { buildList.each { String buildFile -> // check for file level overwrite - if (props.getFileProperty('loadFileLevelProperties', buildFile).toBoolean()) { + loadFileLevelProperties = props.getFileProperty('loadFileLevelProperties', buildFile) + if (loadFileLevelProperties && loadFileLevelProperties.toBoolean()) { String member = new File(buildFile).getName() String propertyFilePath = props.getFileProperty('propertyFilePath', buildFile)