From ab7099ea34b5b757f943704745e194fe1e0c549a Mon Sep 17 00:00:00 2001 From: Wayne Mathis Date: Thu, 5 Dec 2019 16:06:13 -0500 Subject: [PATCH] Removed MappingExtension sample since it is now in the base import tool * DBB Issue 708 * DBB Issue 708 additional removal * DBB Issue 708 modified README --- .../DetectPossibleRoundTripProblems.groovy | 66 ----- Migration/mappingExtension/README.md | 70 ------ Migration/mappingExtension/migrate.groovy | 228 ------------------ README.md | 1 - 4 files changed, 365 deletions(-) delete mode 100644 Migration/mappingExtension/DetectPossibleRoundTripProblems.groovy delete mode 100644 Migration/mappingExtension/README.md delete mode 100644 Migration/mappingExtension/migrate.groovy diff --git a/Migration/mappingExtension/DetectPossibleRoundTripProblems.groovy b/Migration/mappingExtension/DetectPossibleRoundTripProblems.groovy deleted file mode 100644 index 8bb19088..00000000 --- a/Migration/mappingExtension/DetectPossibleRoundTripProblems.groovy +++ /dev/null @@ -1,66 +0,0 @@ -import com.ibm.jzos.* - -class DetectPossibleRoundTripProblems -{ - def CHAR_NL = 0x15 - def CHAR_CR = 0x0D - def CHAR_LF = 0x25 - def CHAR_SHIFT_IN = 0x0F - def CHAR_SHIFT_OUT = 0x0E - - /** - * Detect whether a member contains record that contains a line separator or an empty Shift-In Shift-Out - * @param dataset the data set contains the member to test - * @param member the member to test - * @return an array contains error code 0 = No Errors round, 1 = Contains 1 of the errors described in - * the above description, 2 = Internal Error; and the actual error message. - */ - def call(String dataset, String member) - { - def fullyQualifiedDsn = constructDatasetForZFileOperation(dataset, member) - try - { - def reader = RecordReader.newReader(fullyQualifiedDsn, ZFileConstants.FLAG_DISP_SHR) - byte[] buf = new byte[reader.getLrecl()] - int line = 0 - while (reader.read(buf) >= 0) - { - line++ - def prevIndex = -1 - def foundEmptyShiftOutShiftIn = false - - /* Find NL, CR, LF in a record */ - def found = buf.find { it == CHAR_NL || it == CHAR_CR || it == CHAR_LF } - if (found) - return [1, "Line $line contains a line separator 0x${sprintf('%02X',found)}"] - - /* Find an empty Shift In and Shift Out */ - buf.eachWithIndex {nextByte, nextIndex -> - if (nextByte == CHAR_SHIFT_IN) - prevIndex = nextIndex - else if (nextByte == CHAR_SHIFT_OUT && prevIndex != -1 && prevIndex == (nextIndex-1)) - foundEmptyShiftOutShiftIn = true - } - if (foundEmptyShiftOutShiftIn) - return [1, "Line $line contains empty Shift In and Shift Out"] - } - } - catch (IOException e) - { - return [2, "Internal error ${e.message}"] - } - return [0, "SUCCESS"] - } - - /** - * All zFile operations require dataset and member in certain format - * @param dataset the data set - * @param member the member in the data set - * @return a formatted string contains the data set and member for zFile - * operation - */ - def constructDatasetForZFileOperation(String dataset, String member) - { - return "//'${dataset}($member)'" - } -} diff --git a/Migration/mappingExtension/README.md b/Migration/mappingExtension/README.md deleted file mode 100644 index 78c9547d..00000000 --- a/Migration/mappingExtension/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Migration Sample -The following sample shows how to modify the migration Groovy script to add additional functionality. - -## Requirement -Detect possible round-trip encoding problems when importing and loading to HFS. In this sample, we will address 2 issues: -1. New line character in the record. -2. Empty Shift-In and Shift-Out. - -## Background -Migration tool assists user to import members in data set to files in HFS. User can then deliver these files to the Git repository. Migration does not do any encoding conversion when migrating members to HFS, so if members are encoded in EBCDIC Cp1047, the files in HFS are also encoded in Cp1047. However files stored in distributed Git repository are usually encoded in UTF-8 specially source files. When we mention round-trip conversion, we are referring to the process of converting character set from Cp1047 to UTF-8 when committing to Git and converting it back from UTF-8 to Cp1047 when loading to HFS from the Git repository. There are some situations where this round-trip conversion does not preserve the original content of the source files. For example: -1. **New line character in the record** -The record can contain new line separators (0x15=NL; 0D=CR; 0x25=LF). When committing such files to Git, these new line characters are converted to a brand new line. Therefore, after the round trip conversion, the loaded members would have additional empty record compare to the original migrated member. - -2. **Empty Shift-In and Shift-Out** -The Shift-In (0x0F) and Shift-Out (0x0E) are often used to embed different character sets in a text file. For example, embed a double-byte character sets in a source file. When committing such files to Git, these characters are stripped off and the double-byte character sets are converted. When loading these files from Git to HFS, the Shift-In and Shift-Out are re-inserted. In the case of an empty Shift-In and Shift-Out (0F0E), these characters are stripped off but are never re-inserted later. Again, the content of loaded member would not be exactly the same as the original migrated member. - -## Solution -Provide additional pre-process to detect these scenarios and take appropriate actions during migration. - -Implementing this pre-process in Groovy is straight forward, see sample DetectPossibleRoundTripProblems.groovy. - -> /* Find NL, CR, LF in a record */ -> def found = buf.find { it == CHAR_NL || it == CHAR_CR || it == CHAR_LF } -> if (found) -> return [1, "Line $line contains a line separator 0x${sprintf('%02X',found)}"] - -and - -> /* Find an empty Shift In and Shift Out */ -> buf.eachWithIndex { nextByte, nextIndex -> -> if (nextByte == CHAR_SHIFT_IN) -> prevIndex = nextIndex -> else if (nextByte == CHAR_SHIFT_OUT && prevIndex != -1 && prevIndex == (nextIndex-1)) -> foundEmptyShiftOutShiftIn = true -> } -> if (foundEmptyShiftOutShiftIn) -> return [1, "Line $line contains empty Shift In and Shift Out"] - -The above snippets show how we detect new line characters and empty Shift-In and Shift-Out after reading in the record. In this sample, we return an error code (0=No Failure; 1=Error detected with round-trip encoding) and an error message. The next step is to incorporate this pre-process into the existing migrate.groovy. In this example, the action we will take when encountering these scenarios is to continue migrating these members but flagging this file in the **.gitattributes** file as 'binary' so that there is no EBCDIC to UTF-8 conversion when committing to Git repository. - -First, adding a reference of this script in migrate.groovy: - -> def detector = new DetectPossibleRoundTripProblems() - -Secondly, call this method on each member being migrated: - -> def (rc, msg) = detector(mappingInfo.dataset, mappingInfo.member) - -Lastly, check the rc and flagging this file in .gitattributes when one of these scenarios are detected: - -> if (rc) -> gitAttributeLine = repository.toPath().relativize(hfsFile.toPath()).toFile().path + " binary" - -In this sample, we migrate these particular files to the same directory as other files, but you might want to move these files to a particular 'bin' directory, in that case, simply changing the following in migrate.groovy: - -> new CopyToHFS().dataset(mappingInfo.dataset).member(mappingInfo.member).file(new File(mappingInfo.hfsPath)).pdsEncoding(mappingInfo.pdsEncoding).execute(); - -to - -> def hfsFile = new File(mappingInfo.hfsPath) -> def newHfsPath = "${hfsFile.parent.absolutePath}/bin/${hfsFile.name}" -> new CopyToHFS().dataset(mappingInfo.dataset).member(mappingInfo.member).file(new File(newHfsPath)).pdsEncoding(mappingInfo.pdsEncoding).execute(); - -## Attachment -Completed code sample is in the attached DetectPossibleRoundTripProblems.groovy and Migrate.groovy. - - - - - diff --git a/Migration/mappingExtension/migrate.groovy b/Migration/mappingExtension/migrate.groovy deleted file mode 100644 index 178db0a9..00000000 --- a/Migration/mappingExtension/migrate.groovy +++ /dev/null @@ -1,228 +0,0 @@ -import com.ibm.dbb.migrate.* -import com.ibm.dbb.build.* -import java.io.File - -/********************************************************************************** -usage: migrate [options] | -Use this migration tool to migrate members from data sets to a local GIT -repository on HFS - -m,--mapping The ID of mapping rule (optional), for - example: - com.ibm.dbb.migration.SimpleMapping, - com.ibm.dbb.migration.HlqMapping - -o,--output Output of the generated mapping file - (optional) - -p,--preview Perform a dry-run to generate a mapping - file (optional) - -r,--repository Local GIT repository to migrate to - (required) -***********************************************************************************/ - -def headerMsg = 'Use this migration tool to migrate members from data sets to a local GIT repository on HFS' -cli = new CliBuilder(usage:'migrate [options] ', header: headerMsg, stopAtNonOption: false) -cli.r(longOpt:'repository', args:1, argName:'repository', 'Local GIT repository to migrate to (required)') -cli.o(longOpt:'output', args:1, argName:'output', 'Output of the generated mapping file (optional)') -cli.m(longOpt:'mapping', args:1, argName:'mapping', 'The ID of mapping rule (optional), for example: com.ibm.dbb.migration.MappingRule') -cli.p(longOpt:'preview', 'Perform a dry-run to generate a mapping file (optional)') - -def parameters = cli.parse(args) - -//Validate that we have atleast an argument which is either a set of data sets to be imported or a mapping file -if (parameters.arguments().size() != 1) -{ - cli.usage() - System.exit(2) -} -def arg = parameters.arguments()[0] -boolean isMappingFileSpecified = isMappingFileSpecified(arg) - -def gitAttributePathCache = [] -if (isMappingFileSpecified) -{ - def repository = (parameters.r) ? new File(parameters.r) : null - def gitAttributeWriter = null - if (repository) - { - println("Local GIT repository: $repository") - repository.mkdirs() - gitAttributeWriter = new File(repository, ".gitattributes").newWriter(true) - } - - def detector = new DetectPossibleRoundTripProblems() - - arg.split(",").each { path -> - def mappingFile = new File(path) - println("Migrate data sets using mapping file $mappingFile") - mappingFile.eachLine { line -> - //Ignore comment which starts with # - if (!line.trim().startsWith("#")) - { - //Each line should be in the form of: "dataset hfsPath pdsEncoding=Cp1047" - def lineSegments = line.split(" ") - if (lineSegments.size() >= 2) - { - def datasetMember = lineSegments[0] - boolean isValidDatasetMember = datasetMember ==~ ".*\\((.*[a-zA-Z\\*].*)\\)" - if (isValidDatasetMember) - { - def datasetMemberSegment = datasetMember.split("[\\(\\)]") - def dataset = datasetMemberSegment[0] - def member = datasetMemberSegment[1] - def hfsPath = lineSegments[1]; - def pdsEncoding = null; - if (lineSegments.size() > 2) - { - def pdsEncodingSegments = lineSegments[2].split("=") - if (pdsEncodingSegments.size() == 2 && pdsEncodingSegments[0] == 'pdsEncoding') - pdsEncoding = pdsEncodingSegments[1].trim() - } - def encodingString = pdsEncoding ?: 'default encoding' - println("Copying $datasetMemberSegment to $hfsPath using $encodingString") - - def (rc, msg) = detector(dataset, member) - if (rc) - println("Detecting possible migration in member $member. Actual error is $msg") - - new CopyToHFS().dataset(dataset).member(member).file(new File(hfsPath)).pdsEncoding(pdsEncoding).execute(); - if (gitAttributeWriter) - { - String gitAttributeLine = null; - if (rc) - gitAttributeLine = repository.toPath().relativize(new File(hfsPath).toPath()).toFile().path + " binary" - else - gitAttributeLine = generateGitAttributeEncodingLine(repository, new File(hfsPath), gitAttributePathCache) - if (gitAttributeLine) - gitAttributeWriter.writeLine(gitAttributeLine) - } - } - } - } - } - } - gitAttributeWriter?.close() -} -else -{ - def datasets = arg.split(",") - - //Verify that 'repository' option is specified - if (!parameters.r) - { - cli.usage() - System.exit(2) - } - - //Create the repository directory if not exist - def repository = new File(parameters.r) - if (!repository.exists()) - repository.mkdirs() - println("Local GIT repository: $repository") - - def isPreview = parameters.p - if (isPreview) - println("Preview flag is specified, no members will be copied to HFS") - - //Verify if mapping rule is defined and parse it, otherwise use default 'com.ibm.dbb.migration.SimpleMapping' - def (mappingRuleId,mappingRuleAttrs) = parameters.m ? parseMappingRule(parameters.m) : ['com.ibm.dbb.migration.MappingRule',null] - def mappingRule = (mappingRuleId as Class).newInstance(repository,mappingRuleAttrs) - - println("Using mapping rule $mappingRuleId to migrate the data sets") - - //Verify if an output file is specified to record the mapping - def writer = null; - if (parameters.o) - { - def outputFile = new File(parameters.o) - writer = outputFile.newWriter(true) - println("Generated mappings will be saved in $outputFile") - } - - def gitAttributeWriter = !isPreview ? new File(repository, ".gitattributes").newWriter(true) : null - - println("Start migration...") - - def detector = new DetectPossibleRoundTripProblems() - - datasets.each { dataset -> - println("Migrating data set $dataset") - def mappingInfos = mappingRule.generateMapping(dataset) - mappingInfos.each { mappingInfo -> - if (!isPreview) - { - def encodingString = mappingInfo.pdsEncoding ?: 'default encoding' - println("Copying ${mappingInfo.getFullyQualifiedDsn()} to ${mappingInfo.hfsPath} using $encodingString") - def (rc, msg) = detector(mappingInfo.dataset, mappingInfo.member) - if (rc) - println("Detecting possible migration in member ${mappingInfo.member}. Actual error is $msg") - new CopyToHFS().dataset(mappingInfo.dataset).member(mappingInfo.member).file(new File(mappingInfo.hfsPath)).pdsEncoding(mappingInfo.pdsEncoding).execute(); - if (gitAttributeWriter) - { - def hfsFile = new File(mappingInfo.hfsPath) - String gitAttributeLine = null - if (rc) - gitAttributeLine = repository.toPath().relativize(hfsFile.toPath()).toFile().path + " binary" - else - gitAttributeLine = generateGitAttributeEncodingLine(repository, new File(mappingInfo.hfsPath), gitAttributePathCache) - if (gitAttributeLine) - gitAttributeWriter.writeLine(gitAttributeLine) - } - } - if (writer) - writer.writeLine(mappingInfo.toString()) - } - } - - writer?.close() - gitAttributeWriter?.close() -} - -def parseMappingRule(String mappingRuleId) -{ - def mappingIds = ['MappingRule':'com.ibm.dbb.migration.MappingRule', 'com.ibm.dbb.migration.MappingRule':'com.ibm.dbb.migration.MappingRule'] - def temp = mappingRuleId.split("[\\[\\]]") - if (temp.length == 1) - { - return [temp[0], [:]] - } - else if (temp.length == 2) - { - def id = temp[0] - id = mappingIds[id] - def str = temp[1] - def attrMap = [:] - def attrStrs = str.split(",") - attrStrs.each { attrStr -> - def attr = attrStr.split(":") - def attrName = attr[0] - def attrValue = attr[1] - if (attrValue.startsWith("\"") && attrValue.endsWith("\"") && attrValue.length() > 2) - attrValue = attrValue.substring(1,attrValue.length()-1) - attrMap.put(attrName, attrValue) - } - return [id,attrMap] - } -} - -def generateGitAttributeEncodingLine(File root, File file, List pathCache, String encoding = 'ibm-1047') -{ - def relPath = root.toPath().relativize(file.parentFile.toPath()).toFile() - def index = file.name.lastIndexOf(".") - def fileExtension = (index == -1 || index == (file.name.length() - 1)) ? null : file.name.substring(index+1) - def extension = "*." + (fileExtension ?: "*") - def path = relPath.path + '/' + extension - if (pathCache.contains(path)) - return null - pathCache.add(path) - return path + " working-tree-encoding=${encoding} git-encoding=utf-8" -} - -def isMappingFileSpecified(String argString) -{ - def filesExist = true - argString.split(",").each { path -> - def file = new File(path) - if (!file.exists()) - filesExist = false - } - return filesExist -} diff --git a/README.md b/README.md index 41312b44..ffadcb8e 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ Sample | Description [Build/PublishLoadModules](Build/PublishLoadModules) | Sample demonstrating how to publish load modules to Artifactory after a successful build. [Build/zAppBuild](Build/zAppBuild) | zAppBuild is a generic build solution for building z/OS applications using Apache Groovy build scripts and IBM Dependency Based Build (DBB) APIs. [IDE/GitISPFClient](IDE/GitISPFClient) | An ISPF interface that interacts with a Git repository to allow cloning, staging, checking in, pushing and pulling as well as other git commands. -[Migration/mappingExtension](Migration/mappingExtension) | Sample showing how to extend the DBB Migration tool script to detect possible round-trip encoding problems when importing and loading to HFS. [Migration/sclm](Migration/sclm) | This sample provides scripts to migrate source members to local Git repository and convert the build information in SCLM into build Groovy scripts. [Snippets/InteractiveGateway](Snippets/InteractiveGateway) | Example showing how to use the new ISPFExec/TSOExec Interactive Gateway support added in DBB v1.0.2 [Snippets/PropertyMappings](Snippets/PropertyMappings) | Example showing how to use the new PropertyMappings class to perform aggregate functions on DBB BuildProperties.