From 56aef4d7f47bfdf9025b2f2bda249f8ac93ae3a2 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Thu, 14 Feb 2019 11:22:51 +0100 Subject: [PATCH 01/13] Code style improvements Signed-off-by: Jakob-Naucke --- PublishLoadModules/ArtifactoryHelpers.groovy | 18 +++++++++--------- PublishLoadModules/PublishLoadModule.groovy | 16 ++++++---------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/PublishLoadModules/ArtifactoryHelpers.groovy b/PublishLoadModules/ArtifactoryHelpers.groovy index b5224df..bc4b456 100644 --- a/PublishLoadModules/ArtifactoryHelpers.groovy +++ b/PublishLoadModules/ArtifactoryHelpers.groovy @@ -29,31 +29,30 @@ import groovyx.net.http.* /** * Publish a file from HFS to an artifactory repository at location specified in remoteFilePath */ -def publish(serverUrl, repo, apiKey, remoteFilePath, File localFile) -{ +def publish(serverUrl, repo, apiKey, remoteFilePath, File localFile) { //Validate to make sure all required fields are specified assert serverUrl != null, "Need to specify a valid URL to artifactory server" assert repo != null, "Need to specify a valid artifactory repository" assert apiKey != null, "Need to specify a valid API key to authenticate with $repo" assert remoteFilePath != null, "Need to specify the path of the source file" assert localFile != null && localFile.exists(), "Target local file must exist" - + //Artifactory URL must end with '/' def url = serverUrl.endsWith('/') ? serverUrl : serverUrl + '/' //Create SHA1 and MD5 checksums to be published along with the file def sha1 = getChecksum(localFile) def md5 = getChecksum(localFile, "MD5") - - def filePath = "$repo/$remoteFilePath" - + + def filePath = "$repo/$remoteFilePath" + def restClient = new RESTClient(url) restClient.encoder.'application/zip' = this.&encodeZipFile def response = restClient.put(path: filePath, headers: ['X-JFrog-Art-Api' : apiKey, 'X-Checksum-Sha1' : sha1, 'X-Checksum-MD5' : md5], body: localFile, requestContentType: 'application/zip') - + assert response.isSuccess(), "Failed to publish file $localFile" - - println "Successfully publish file $localFile to $filePath" + + println "Successfully published file $localFile to $filePath" } /** @@ -137,3 +136,4 @@ def static encodeZipFile(Object data) throws UnsupportedEncodingException entity.setContentType('application/zip'); return entity } + diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index 1b85863..a9ad706 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -9,8 +9,8 @@ import com.ibm.dbb.build.report.records.DefaultRecordFactory import groovyx.net.http.RESTClient /************************************************************************************ - * This script publishes the outputs generated from a build to an artifactory - * repository. + * This script publishes the outputs generated from a build to an Artifactory + * repository. * ************************************************************************************/ @@ -79,18 +79,14 @@ int rc = process.waitFor() assert rc == 0, "Failed to package load modules" //Set up the artifactory information to publish the tar file -def url = properties.get('artifactory.url') -def apiKey = properties.get('artifactory.apiKey') -def repo = properties.get('artifactory.repo') as String +def artifactoryURL = properties.get("artifactory.url") as String +def artifactoryRepo = properties.get("artifactory.repo") as String +def artifactoryKey = properties.get("artifactory.apiKey") as String def remotePath = "${buildGroup}/${tarFile.name}" //Call the ArtifactoryHelpers to publish the tar file File artifactoryHelpersFile = new File('./ArtifactoryHelpers.groovy') Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile) GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance() -artifactoryHelpers.publish(url, repo, apiKey, remotePath, tarFile) - - - - +artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, remotePath, tarFile) From 23e564217ada49291e847099b4f29d554ad2925e Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Thu, 14 Feb 2019 11:36:18 +0100 Subject: [PATCH 02/13] Fixed checksum calculation (leading zeroes) Signed-off-by: Jakob-Naucke --- PublishLoadModules/ArtifactoryHelpers.groovy | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/PublishLoadModules/ArtifactoryHelpers.groovy b/PublishLoadModules/ArtifactoryHelpers.groovy index bc4b456..bdb5228 100644 --- a/PublishLoadModules/ArtifactoryHelpers.groovy +++ b/PublishLoadModules/ArtifactoryHelpers.groovy @@ -41,7 +41,7 @@ def publish(serverUrl, repo, apiKey, remoteFilePath, File localFile) { def url = serverUrl.endsWith('/') ? serverUrl : serverUrl + '/' //Create SHA1 and MD5 checksums to be published along with the file - def sha1 = getChecksum(localFile) + def sha1 = getChecksum(localFile, "SHA1") def md5 = getChecksum(localFile, "MD5") def filePath = "$repo/$remoteFilePath" @@ -85,8 +85,8 @@ def download(serverUrl, repo, apiKey, remoteFilePath, File localFile) //the transfer is complete def expectedSha1 = response.headers['X-Checksum-Sha1'].value def expectedMd5 = response.headers['X-Checksum-Md5'].value - def actualSha1 = getChecksum(localFile) - def actualMd5 = getChecksum(localFile, "MD5") + def actualSha1 = getChecksum(localFile, "SHA1") + def actualMd5 = getChecksum(localFile, "MD5") assert actualSha1 == expectedSha1 && actualMd5 == expectedMd5, "The downloaded file $localFile does not have the right checksum" println "Successfully download $filePath to $localFile" @@ -127,7 +127,11 @@ def getChecksum(File file, type = 'SHA1') def digest = MessageDigest.getInstance(type) digest.update(file.bytes) - return new BigInteger(1,digest.digest()).toString(16) + switch (type) { + case "SHA1": return new BigInteger(1,digest.digest()).toString(16).padLeft(40, '0'); break + case "MD5": return new BigInteger(1,digest.digest()).toString(16).padLeft(32, '0'); break + default: println "Unsupported type" + } } def static encodeZipFile(Object data) throws UnsupportedEncodingException From 9faaadcfa01d921e868fb9f5d5994572f061b66b Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Thu, 14 Feb 2019 11:38:21 +0100 Subject: [PATCH 03/13] Fixed taring (tar used to include itself) Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index a9ad706..098a998 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -74,9 +74,7 @@ loadDatasetToMembersMap.each { dataset, members -> def buildGroup = "${properties.collection}" as String def buildLabel = "build.${properties.startTime}" as String def tarFile = new File("$tempLoadDir/${buildLabel}.tar") -def process = "tar -cvf $tarFile .".execute(null, tempLoadDir) -int rc = process.waitFor() -assert rc == 0, "Failed to package load modules" +def tarOut = ["sh", "-c", "cd $tempLoadDir && tar cf $tarFile *"].execute().text //Set up the artifactory information to publish the tar file def artifactoryURL = properties.get("artifactory.url") as String From 562258fc185316d8305621caef855bffcf0a56b2 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Thu, 14 Feb 2019 11:41:24 +0100 Subject: [PATCH 04/13] Fixed overwriting load modules with DBRMs (now dedicated directories) Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index 098a998..b9c85d9 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -60,12 +60,16 @@ tempLoadDir.mkdirs() //SSI and CopyToHFS copy = new CopyToHFS().copyMode(CopyMode.LOAD) println "Number of load modules to publish: $loadCount" + +// Create dedicated directories for datasets (e.g. load modules and DBRMs) loadDatasetToMembersMap.each { dataset, members -> + datasetDir = new File("$tempLoadDir/$dataset") + datasetDir.mkdirs() members.each { member -> def fullyQualifiedDsn = "$dataset($member)" - def file = new File(tempLoadDir, member) + def file = new File(datasetDir, member) copy.dataset(dataset).member(member).file(file).copy() - println "Copying $dataset($member) to $tempLoadDir" + println "Copying $dataset($member) to $datasetDir" } } From 768b7f7e51296a7efe56597c603bb8c70bd7c33b Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Thu, 14 Feb 2019 11:42:41 +0100 Subject: [PATCH 05/13] Switched setup for the General-Insurance application Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 25 ++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index b9c85d9..e155b10 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -1,6 +1,8 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript import java.io.File import java.io.UnsupportedEncodingException import java.security.MessageDigest +import java.text.SimpleDateFormat import org.apache.http.entity.FileEntity import com.ibm.dbb.build.* import com.ibm.dbb.build.DBBConstants.CopyMode @@ -14,7 +16,16 @@ import groovyx.net.http.RESTClient * ************************************************************************************/ -def properties = BuildProperties.getInstance() +def currentDir = new File(getClass().protectionDomain.codeSource.location.path).parent + +// load the Tools.groovy utility script +def tools = loadScript(new File("$currentDir/Tools.groovy")) + +// parse command line arguments and load build properties +def usage = "PublishLoadModule.groovy [options]" +def opts = tools.parseArgs(args, usage) +def properties = tools.loadProperties(opts) + def workDir = properties.workDir def loadDatasets = properties.loadDatasets @@ -73,10 +84,18 @@ loadDatasetToMembersMap.each { dataset, members -> } } +// Append build report +def exportBuildReport = new File("$tempLoadDir/BuildReport.json") +exportBuildReport << buildReportFile.text + +def date = new Date() +def sdf = new SimpleDateFormat("yyyyMMdd-HHmmss") +def startTime = sdf.format(date) as String + //Package the load files just copied into a tar file using the build //label as the name for the tar file. def buildGroup = "${properties.collection}" as String -def buildLabel = "build.${properties.startTime}" as String +def buildLabel = "build.$startTime" as String def tarFile = new File("$tempLoadDir/${buildLabel}.tar") def tarOut = ["sh", "-c", "cd $tempLoadDir && tar cf $tarFile *"].execute().text @@ -87,7 +106,7 @@ def artifactoryKey = properties.get("artifactory.apiKey") as String def remotePath = "${buildGroup}/${tarFile.name}" //Call the ArtifactoryHelpers to publish the tar file -File artifactoryHelpersFile = new File('./ArtifactoryHelpers.groovy') +File artifactoryHelpersFile = new File("$currentDir/ArtifactoryHelpers.groovy") Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile) GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance() artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, remotePath, tarFile) From 87e5f43d00d994e462d41d586680e89c0a17b1b7 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Thu, 14 Feb 2019 12:52:53 +0100 Subject: [PATCH 06/13] Changed Publisher README from Mortgage to General-Insurance Signed-off-by: Jakob-Naucke --- PublishLoadModules/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PublishLoadModules/README.md b/PublishLoadModules/README.md index 2e18ede..6ae38bf 100644 --- a/PublishLoadModules/README.md +++ b/PublishLoadModules/README.md @@ -2,10 +2,10 @@ This sample shows how to publish load modules to an artifactory repository after a successful build, as well as download load modules from the artifactory repository and restore them into an existing data set. Since all of interaction with artifactory repository requires files on zFS, load modules need to copy from data set to files on zFS and vice-versa. This sample therefore also makes use of the new options introduced in CopyToPDS and CopyToHFS APIs to copy between data set and files on zFS. ## Prerequisites: -This sample is built on top of the Mortgage Application Sample, so it requires a successful Mortgage setup. It also requires a set of jar files which can be downloaded from Maven Central Repository. These jar files are required for making REST service calls to Artifactory Repository using Groovy, see ArtifactoryHelpers.groovy for more details. +This sample is built on top of the General-Insurance Application Sample, so it requires a successful General-Insurance setup. It also requires a set of jar files which can be downloaded from Maven Central Repository. These jar files are required for making REST service calls to Artifactory Repository using Groovy, see ArtifactoryHelpers.groovy for more details. ## Scenario 1 - Publishing load modules from a successful build -1. After a successful Mortgage build, it retrieves all outputs from the build report. +1. After a successful General-Insurance build, it retrieves all outputs from the build report. 2. From the list of the outputs, it filters the load modules based on the data sets specified in the build property 'loadDatasets'. For example: the build report could contain outputs from BMS, for example: USER1.DBB.COPYBOOKS(ESPMLIS), USER1.DBB.DBRM(EPSCMORT), USER1.DBB.LOAD(EPSCMORT), but the user is only interested in publishing load modules in USER1.DBB.LOAD. The build property 'loadDatasets' should then be set to 'USER1.DBB.LOAD' 3. It then invokes CopyToHFS to copy the load modules from the PDSe to a temporary directory on zFS. 4. It packages these load files into a tar file, and compute the SHA1 and MD5 checksums. From af2ede578f6a9997e04cef02d6517eb057dca971 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 16:24:16 +0100 Subject: [PATCH 07/13] Style improvements Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 29 ++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index e155b10..b790516 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -16,12 +16,12 @@ import groovyx.net.http.RESTClient * ************************************************************************************/ -def currentDir = new File(getClass().protectionDomain.codeSource.location.path).parent +def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent -// load the Tools.groovy utility script -def tools = loadScript(new File("$currentDir/Tools.groovy")) +// Load the Tools.groovy utility script +def tools = loadScript(new File("$scriptDir/Tools.groovy")) -// parse command line arguments and load build properties +// Parse command line arguments and load build properties def usage = "PublishLoadModule.groovy [options]" def opts = tools.parseArgs(args, usage) def properties = tools.loadProperties(opts) @@ -29,7 +29,7 @@ def properties = tools.loadProperties(opts) def workDir = properties.workDir def loadDatasets = properties.loadDatasets -//Retrieve the build report and parse the outputs from the build report +// Retrieve the build report and parse the outputs from the build report def buildReportFile = new File("$workDir/BuildReport.json") assert buildReportFile.exists(), "$buildReportFile does not exist" @@ -40,8 +40,8 @@ def executes = buildReport.records.findAll { record -> assert executes.size() > 0, "There are no outputs found in the build report" -//If the user specifies the build property 'loadDatasets' then retrieves it -//and filters out only outputs that match with the specified data sets. +// If the user specifies the build property 'loadDatasets' then retrieves it +// and filters out only outputs that match with the specified data sets. def loadDatasetArray = loadDatasets?.split(",") def loadDatasetList = loadDatasetArray == null ? [] : Arrays.asList(loadDatasetArray) @@ -62,13 +62,12 @@ executes.each { execute -> assert loadCount > 0, "There are no load modules to publish" -//Create a temporary directory on zFS to copy the load modules from data sets to +// Create a temporary directory on zFS to copy the load modules from data sets to def tempLoadDir = new File("$workDir/tempLoadDir") !tempLoadDir.exists() ?: tempLoadDir.deleteDir() tempLoadDir.mkdirs() -//For each load modules, use CopyToHFS with option 'CopyMode.LOAD' to maintain -//SSI and +// For each load module, use CopyToHFS with option 'CopyMode.LOAD' to maintain SSI CopyToHFS copy = new CopyToHFS().copyMode(CopyMode.LOAD) println "Number of load modules to publish: $loadCount" @@ -92,21 +91,21 @@ def date = new Date() def sdf = new SimpleDateFormat("yyyyMMdd-HHmmss") def startTime = sdf.format(date) as String -//Package the load files just copied into a tar file using the build -//label as the name for the tar file. +// Package the load files just copied into a tar file using the build +// label as the name for the tar file. def buildGroup = "${properties.collection}" as String def buildLabel = "build.$startTime" as String def tarFile = new File("$tempLoadDir/${buildLabel}.tar") def tarOut = ["sh", "-c", "cd $tempLoadDir && tar cf $tarFile *"].execute().text -//Set up the artifactory information to publish the tar file +// Set up the artifactory information to publish the tar file def artifactoryURL = properties.get("artifactory.url") as String def artifactoryRepo = properties.get("artifactory.repo") as String def artifactoryKey = properties.get("artifactory.apiKey") as String def remotePath = "${buildGroup}/${tarFile.name}" -//Call the ArtifactoryHelpers to publish the tar file -File artifactoryHelpersFile = new File("$currentDir/ArtifactoryHelpers.groovy") +// Call the ArtifactoryHelpers to publish the tar file +File artifactoryHelpersFile = new File("$scriptDir/ArtifactoryHelpers.groovy") Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile) GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance() artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, remotePath, tarFile) From dd69bce192d38057910b602e55a92be9b7b47563 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 16:37:07 +0100 Subject: [PATCH 08/13] Use proper copy modes for CopyToHFS Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index b790516..280bce9 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -8,7 +8,6 @@ import com.ibm.dbb.build.* import com.ibm.dbb.build.DBBConstants.CopyMode import com.ibm.dbb.build.report.BuildReport import com.ibm.dbb.build.report.records.DefaultRecordFactory -import groovyx.net.http.RESTClient /************************************************************************************ * This script publishes the outputs generated from a build to an Artifactory @@ -67,20 +66,25 @@ def tempLoadDir = new File("$workDir/tempLoadDir") !tempLoadDir.exists() ?: tempLoadDir.deleteDir() tempLoadDir.mkdirs() -// For each load module, use CopyToHFS with option 'CopyMode.LOAD' to maintain SSI -CopyToHFS copy = new CopyToHFS().copyMode(CopyMode.LOAD) +// For each load module, use CopyToHFS with respective CopyMode option to maintain SSI +def copy = new CopyToHFS() +def copyModeMap = ["COPYBOOK": CopyMode.TEXT, "DBRM": CopyMode.BINARY, "LOAD": CopyMode.LOAD] println "Number of load modules to publish: $loadCount" // Create dedicated directories for datasets (e.g. load modules and DBRMs) loadDatasetToMembersMap.each { dataset, members -> datasetDir = new File("$tempLoadDir/$dataset") datasetDir.mkdirs() + + currentCopyMode = copyModeMap[dataset.replaceAll(/.*\.([^.]*)/, "\$1")] + copy.setCopyMode(currentCopyMode) + copy.setDataset(dataset) + members.each { member -> - def fullyQualifiedDsn = "$dataset($member)" - def file = new File(datasetDir, member) - copy.dataset(dataset).member(member).file(file).copy() println "Copying $dataset($member) to $datasetDir" + copy.member(member).file(new File("$datasetDir/$member")).copy() } + } // Append build report From 6c12387597c0a4a4d3ade44762a80b64df3c1937 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 16:39:20 +0100 Subject: [PATCH 09/13] Write datasets out to a CSV file Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 28 +++++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index 280bce9..fcd6465 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -71,20 +71,26 @@ def copy = new CopyToHFS() def copyModeMap = ["COPYBOOK": CopyMode.TEXT, "DBRM": CopyMode.BINARY, "LOAD": CopyMode.LOAD] println "Number of load modules to publish: $loadCount" -// Create dedicated directories for datasets (e.g. load modules and DBRMs) -loadDatasetToMembersMap.each { dataset, members -> - datasetDir = new File("$tempLoadDir/$dataset") - datasetDir.mkdirs() +// Create a file to specify datasets +def datasetsCSV = new File("$tempLoadDir/Datasets.csv") - currentCopyMode = copyModeMap[dataset.replaceAll(/.*\.([^.]*)/, "\$1")] - copy.setCopyMode(currentCopyMode) - copy.setDataset(dataset) +// Create dedicated directories for datasets (e.g. load modules and DBRMs) +datasetsCSV.withWriter("UTF-8") { writer -> + loadDatasetToMembersMap.each { dataset, members -> + datasetDir = new File("$tempLoadDir/$dataset") + datasetDir.mkdirs() + + currentCopyMode = copyModeMap[dataset.replaceAll(/.*\.([^.]*)/, "\$1")] + copy.setCopyMode(currentCopyMode) + copy.setDataset(dataset) + + members.each { member -> + println "Copying $dataset($member) to $datasetDir" + copy.member(member).file(new File("$datasetDir/$member")).copy() + } - members.each { member -> - println "Copying $dataset($member) to $datasetDir" - copy.member(member).file(new File("$datasetDir/$member")).copy() + writer.writeLine dataset } - } // Append build report From 7b04e728e20d919cf451f76c7db92eb166a8fb16 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 16:40:29 +0100 Subject: [PATCH 10/13] Append logs Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index fcd6465..160a236 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -96,6 +96,13 @@ datasetsCSV.withWriter("UTF-8") { writer -> // Append build report def exportBuildReport = new File("$tempLoadDir/BuildReport.json") exportBuildReport << buildReportFile.text +// Append all log files +def logDirectory = new File("$tempLoadDir/Logs") +logDirectory.mkdirs() +new File(workDir).eachFileMatch(~/.*\.log/) { logFile -> + copiedLogFile = new File("$logDirectory/$logFile.name") + copiedLogFile << logFile.text +} def date = new Date() def sdf = new SimpleDateFormat("yyyyMMdd-HHmmss") From dbec506c987bfcd57b33c44ad83f1f17e2f33ef7 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 17:04:50 +0100 Subject: [PATCH 11/13] Streamlined variables relevant to ArtifactoryHelpers Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index 160a236..15eaa07 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -104,26 +104,26 @@ new File(workDir).eachFileMatch(~/.*\.log/) { logFile -> copiedLogFile << logFile.text } +// Get date for version label def date = new Date() def sdf = new SimpleDateFormat("yyyyMMdd-HHmmss") def startTime = sdf.format(date) as String // Package the load files just copied into a tar file using the build -// label as the name for the tar file. -def buildGroup = "${properties.collection}" as String -def buildLabel = "build.$startTime" as String +// label as the name for the tar file +def buildLabel = "build.$startTime" def tarFile = new File("$tempLoadDir/${buildLabel}.tar") def tarOut = ["sh", "-c", "cd $tempLoadDir && tar cf $tarFile *"].execute().text -// Set up the artifactory information to publish the tar file +// Set up the Artifactory information to publish the tar file def artifactoryURL = properties.get("artifactory.url") as String def artifactoryRepo = properties.get("artifactory.repo") as String def artifactoryKey = properties.get("artifactory.apiKey") as String -def remotePath = "${buildGroup}/${tarFile.name}" +def artifactoryComponent = properties.get("collection") as String // Call the ArtifactoryHelpers to publish the tar file File artifactoryHelpersFile = new File("$scriptDir/ArtifactoryHelpers.groovy") Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile) GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance() -artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, remotePath, tarFile) +artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, "$artifactoryComponent/$tarFile.name", tarFile) From dc9dfb214bb9978b7e2d4a0eb9cf2d8934d45db1 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 17:08:33 +0100 Subject: [PATCH 12/13] Improved handling of tar process Signed-off-by: Jakob-Naucke --- PublishLoadModules/PublishLoadModule.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index 15eaa07..76362d7 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -113,7 +113,8 @@ def startTime = sdf.format(date) as String // label as the name for the tar file def buildLabel = "build.$startTime" def tarFile = new File("$tempLoadDir/${buildLabel}.tar") -def tarOut = ["sh", "-c", "cd $tempLoadDir && tar cf $tarFile *"].execute().text +def process = ["sh", "-c", "tar cf $tarFile *"].execute([], tempLoadDir) +assert process.waitFor() == 0, "Failed to package" // Set up the Artifactory information to publish the tar file def artifactoryURL = properties.get("artifactory.url") as String From 86bb26d3cb34744522f618c01966accec187ade8 Mon Sep 17 00:00:00 2001 From: Jakob-Naucke Date: Fri, 15 Mar 2019 17:09:36 +0100 Subject: [PATCH 13/13] Fetch URL from Artifactory Signed-off-by: Jakob-Naucke --- PublishLoadModules/ArtifactoryHelpers.groovy | 6 ++++++ PublishLoadModules/PublishLoadModule.groovy | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/PublishLoadModules/ArtifactoryHelpers.groovy b/PublishLoadModules/ArtifactoryHelpers.groovy index bdb5228..98e22e5 100644 --- a/PublishLoadModules/ArtifactoryHelpers.groovy +++ b/PublishLoadModules/ArtifactoryHelpers.groovy @@ -1,6 +1,7 @@ import java.security.MessageDigest import org.apache.http.entity.FileEntity import groovyx.net.http.* +import groovy.json.JsonSlurper /************************************************************************************ * @@ -47,12 +48,17 @@ def publish(serverUrl, repo, apiKey, remoteFilePath, File localFile) { def filePath = "$repo/$remoteFilePath" def restClient = new RESTClient(url) + restClient.encoderRegistry = new EncoderRegistry(charset: "UTF-8") restClient.encoder.'application/zip' = this.&encodeZipFile def response = restClient.put(path: filePath, headers: ['X-JFrog-Art-Api' : apiKey, 'X-Checksum-Sha1' : sha1, 'X-Checksum-MD5' : md5], body: localFile, requestContentType: 'application/zip') assert response.isSuccess(), "Failed to publish file $localFile" + def jsonSlurper = new JsonSlurper() + def pullableURI = jsonSlurper.parseText(response.data.getText("UTF-8")).uri + assert pullableURI != null: "Artifactory did not return a URI" println "Successfully published file $localFile to $filePath" + return pullableURI } /** diff --git a/PublishLoadModules/PublishLoadModule.groovy b/PublishLoadModules/PublishLoadModule.groovy index 76362d7..f8aa414 100644 --- a/PublishLoadModules/PublishLoadModule.groovy +++ b/PublishLoadModules/PublishLoadModule.groovy @@ -126,5 +126,5 @@ def artifactoryComponent = properties.get("collection") as String File artifactoryHelpersFile = new File("$scriptDir/ArtifactoryHelpers.groovy") Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile) GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance() -artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, "$artifactoryComponent/$tarFile.name", tarFile) +def artifactoryPullableURL = artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, "$artifactoryComponent/$tarFile.name", tarFile)