Bugfixes & adjustment to General-Insurance for publishing DBB outputs

PublishLoadModules/ArtifactoryHelpers.groovy

@@ -1,6 +1,7 @@
 import java.security.MessageDigest
 import org.apache.http.entity.FileEntity
 import groovyx.net.http.*
+import groovy.json.JsonSlurper
 
 /************************************************************************************
  *
@@ -29,31 +30,35 @@ import groovyx.net.http.*
 /**
  * Publish a file from HFS to an artifactory repository at location specified in remoteFilePath
  */
-def publish(serverUrl, repo, apiKey, remoteFilePath, File localFile)
-{
+def publish(serverUrl, repo, apiKey, remoteFilePath, File localFile) {
  //Validate to make sure all required fields are specified
  assert serverUrl != null, "Need to specify a valid URL to artifactory server"
  assert repo != null, "Need to specify a valid artifactory repository"
  assert apiKey != null, "Need to specify a valid API key to authenticate with $repo"
  assert remoteFilePath != null, "Need to specify the path of the source file"
  assert localFile != null && localFile.exists(), "Target local file must exist"
-
+
  //Artifactory URL must end with '/'
  def url = serverUrl.endsWith('/') ? serverUrl : serverUrl + '/'
 
  //Create SHA1 and MD5 checksums to be published along with the file
- def sha1 = getChecksum(localFile)
+ def sha1 = getChecksum(localFile, "SHA1")
  def md5 = getChecksum(localFile, "MD5")
-
- def filePath = "$repo/$remoteFilePath"
-
+
+ def filePath = "$repo/$remoteFilePath"
+
  def restClient = new RESTClient(url)
+ restClient.encoderRegistry = new EncoderRegistry(charset: "UTF-8")
  restClient.encoder.'application/zip' = this.&encodeZipFile
  def response = restClient.put(path: filePath, headers: ['X-JFrog-Art-Api' : apiKey, 'X-Checksum-Sha1' : sha1, 'X-Checksum-MD5' : md5], body: localFile, requestContentType: 'application/zip')
-
+
  assert response.isSuccess(), "Failed to publish file $localFile"
-
- println "Successfully publish file $localFile to $filePath"
+
+ def jsonSlurper = new JsonSlurper()
+ def pullableURI = jsonSlurper.parseText(response.data.getText("UTF-8")).uri
+ assert pullableURI != null: "Artifactory did not return a URI"
+ println "Successfully published file $localFile to $filePath"
+ return pullableURI
 }
 
 /**
@@ -86,8 +91,8 @@ def download(serverUrl, repo, apiKey, remoteFilePath, File localFile)
  //the transfer is complete
  def expectedSha1 = response.headers['X-Checksum-Sha1'].value
  def expectedMd5 = response.headers['X-Checksum-Md5'].value 
- def actualSha1 = getChecksum(localFile)
- def actualMd5 = getChecksum(localFile, "MD5")
+ def actualSha1 = getChecksum(localFile, "SHA1")
+ def actualMd5 = getChecksum(localFile, "MD5")
  assert actualSha1 == expectedSha1 && actualMd5 == expectedMd5, "The downloaded file $localFile does not have the right checksum"
 
  println "Successfully download $filePath to $localFile"
@@ -128,7 +133,11 @@ def getChecksum(File file, type = 'SHA1')
 
  def digest = MessageDigest.getInstance(type)
  digest.update(file.bytes)
- return new BigInteger(1,digest.digest()).toString(16)
+ switch (type) {
+ case "SHA1": return new BigInteger(1,digest.digest()).toString(16).padLeft(40, '0'); break
+ case "MD5": return new BigInteger(1,digest.digest()).toString(16).padLeft(32, '0'); break
+ default: println "Unsupported type"
+ }
 }
 
 def static encodeZipFile(Object data) throws UnsupportedEncodingException
@@ -137,3 +146,4 @@ def static encodeZipFile(Object data) throws UnsupportedEncodingException
  entity.setContentType('application/zip');
  return entity
 }
+

PublishLoadModules/PublishLoadModule.groovy

@@ -1,24 +1,34 @@
+@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript
 import java.io.File
 import java.io.UnsupportedEncodingException
 import java.security.MessageDigest
+import java.text.SimpleDateFormat
 import org.apache.http.entity.FileEntity
 import com.ibm.dbb.build.*
 import com.ibm.dbb.build.DBBConstants.CopyMode
 import com.ibm.dbb.build.report.BuildReport
 import com.ibm.dbb.build.report.records.DefaultRecordFactory
-import groovyx.net.http.RESTClient
 
 /************************************************************************************
- * This script publishes the outputs generated from a build to an artifactory
- * repository.
+ * This script publishes the outputs generated from a build to an Artifactory
+ * repository.
  *
  ************************************************************************************/
 
-def properties = BuildProperties.getInstance()
+def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent
+
+// Load the Tools.groovy utility script
+def tools = loadScript(new File("$scriptDir/Tools.groovy"))
+
+// Parse command line arguments and load build properties
+def usage = "PublishLoadModule.groovy [options]"
+def opts = tools.parseArgs(args, usage)
+def properties = tools.loadProperties(opts)
+
 def workDir = properties.workDir
 def loadDatasets = properties.loadDatasets
 
-//Retrieve the build report and parse the outputs from the build report
+//Retrieve the build report and parse the outputs from the build report
 def buildReportFile = new File("$workDir/BuildReport.json")
 assert buildReportFile.exists(), "$buildReportFile does not exist"
 
@@ -29,8 +39,8 @@ def executes = buildReport.records.findAll { record ->
 
 assert executes.size() > 0, "There are no outputs found in the build report"
 
-//If the user specifies the build property 'loadDatasets' then retrieves it
-//and filters out only outputs that match with the specified data sets.
+//If the user specifies the build property 'loadDatasets' then retrieves it
+//and filters out only outputs that match with the specified data sets.
 def loadDatasetArray = loadDatasets?.split(",")
 def loadDatasetList = loadDatasetArray == null ? [] : Arrays.asList(loadDatasetArray)
 
@@ -51,46 +61,70 @@ executes.each { execute ->
 
 assert loadCount > 0, "There are no load modules to publish"
 
-//Create a temporary directory on zFS to copy the load modules from data sets to
+//Create a temporary directory on zFS to copy the load modules from data sets to
 def tempLoadDir = new File("$workDir/tempLoadDir")
 !tempLoadDir.exists() ?: tempLoadDir.deleteDir()
 tempLoadDir.mkdirs()
 
-//For each load modules, use CopyToHFS with option 'CopyMode.LOAD' to maintain
-//SSI and 
-CopyToHFS copy = new CopyToHFS().copyMode(CopyMode.LOAD)
+//For each load module, use CopyToHFS with respective CopyMode option to maintain SSI
+def copy = new CopyToHFS()
+def copyModeMap = ["COPYBOOK": CopyMode.TEXT, "DBRM": CopyMode.BINARY, "LOAD": CopyMode.LOAD]
 println "Number of load modules to publish: $loadCount"
-loadDatasetToMembersMap.each { dataset, members ->
- members.each { member ->
- def fullyQualifiedDsn = "$dataset($member)"
- def file = new File(tempLoadDir, member)
- copy.dataset(dataset).member(member).file(file).copy()
- println "Copying $dataset($member) to $tempLoadDir"
+
+// Create a file to specify datasets
+def datasetsCSV = new File("$tempLoadDir/Datasets.csv")
+
+// Create dedicated directories for datasets (e.g. load modules and DBRMs)
+datasetsCSV.withWriter("UTF-8") { writer ->
+ loadDatasetToMembersMap.each { dataset, members ->
+ datasetDir = new File("$tempLoadDir/$dataset")
+ datasetDir.mkdirs()
+
+ currentCopyMode = copyModeMap[dataset.replaceAll(/.*\.([^.]*)/, "\$1")]
+ copy.setCopyMode(currentCopyMode)
+ copy.setDataset(dataset)
+
+ members.each { member ->
+ println "Copying $dataset($member) to $datasetDir"
+ copy.member(member).file(new File("$datasetDir/$member")).copy()
+ }
+
+ writer.writeLine dataset
  }
 }
 
-//Package the load files just copied into a tar file using the build
-//label as the name for the tar file.
-def buildGroup = "${properties.collection}" as String
-def buildLabel = "build.${properties.startTime}" as String
-def tarFile = new File("$tempLoadDir/${buildLabel}.tar")
-def process = "tar -cvf $tarFile .".execute(null, tempLoadDir)
-int rc = process.waitFor()
-assert rc == 0, "Failed to package load modules" 
-
-//Set up the artifactory information to publish the tar file
-def url = properties.get('artifactory.url')
-def apiKey = properties.get('artifactory.apiKey')
-def repo = properties.get('artifactory.repo') as String
-def remotePath = "${buildGroup}/${tarFile.name}"
-
-//Call the ArtifactoryHelpers to publish the tar file
-File artifactoryHelpersFile = new File('./ArtifactoryHelpers.groovy')
-Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile)
-GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance()
-artifactoryHelpers.publish(url, repo, apiKey, remotePath, tarFile)
+// Append build report
+def exportBuildReport = new File("$tempLoadDir/BuildReport.json")
+exportBuildReport << buildReportFile.text
+// Append all log files
+def logDirectory = new File("$tempLoadDir/Logs")
+logDirectory.mkdirs()
+new File(workDir).eachFileMatch(~/.*\.log/) { logFile ->
+ copiedLogFile = new File("$logDirectory/$logFile.name")
+ copiedLogFile << logFile.text
+}
 
+// Get date for version label
+def date = new Date()
+def sdf = new SimpleDateFormat("yyyyMMdd-HHmmss")
+def startTime = sdf.format(date) as String
 
+// Package the load files just copied into a tar file using the build
+// label as the name for the tar file
+def buildLabel = "build.$startTime"
+def tarFile = new File("$tempLoadDir/${buildLabel}.tar")
+def process = ["sh", "-c", "tar cf $tarFile *"].execute([], tempLoadDir)
+assert process.waitFor() == 0, "Failed to package"
 
+// Set up the Artifactory information to publish the tar file
+def artifactoryURL = properties.get("artifactory.url") as String
+def artifactoryRepo = properties.get("artifactory.repo") as String
+def artifactoryKey = properties.get("artifactory.apiKey") as String
+def artifactoryComponent = properties.get("collection") as String
 
+// Call the ArtifactoryHelpers to publish the tar file
+File artifactoryHelpersFile = new File("$scriptDir/ArtifactoryHelpers.groovy")
+Class artifactoryHelpersClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(artifactoryHelpersFile)
+GroovyObject artifactoryHelpers = (GroovyObject) artifactoryHelpersClass.newInstance()
+def artifactoryPullableURL = artifactoryHelpers.publish(artifactoryURL, artifactoryRepo, artifactoryKey, "$artifactoryComponent/$tarFile.name", tarFile)
 

PublishLoadModules/README.md

@@ -2,10 +2,10 @@
 This sample shows how to publish load modules to an artifactory repository after a successful build, as well as download load modules from the artifactory repository and restore them into an existing data set. Since all of interaction with artifactory repository requires files on zFS, load modules need to copy from data set to files on zFS and vice-versa. This sample therefore also makes use of the new options introduced in CopyToPDS and CopyToHFS APIs to copy between data set and files on zFS.
 
 ## Prerequisites:
-This sample is built on top of the Mortgage Application Sample, so it requires a successful Mortgage setup. It also requires a set of jar files which can be downloaded from Maven Central Repository. These jar files are required for making REST service calls to Artifactory Repository using Groovy, see ArtifactoryHelpers.groovy for more details. 
+This sample is built on top of the General-Insurance Application Sample, so it requires a successful General-Insurance setup. It also requires a set of jar files which can be downloaded from Maven Central Repository. These jar files are required for making REST service calls to Artifactory Repository using Groovy, see ArtifactoryHelpers.groovy for more details. 
 
 ## Scenario 1 - Publishing load modules from a successful build
-1. After a successful Mortgage build, it retrieves all outputs from the build report.
+1. After a successful General-Insurance build, it retrieves all outputs from the build report.
 2. From the list of the outputs, it filters the load modules based on the data sets specified in the build property 'loadDatasets'. For example: the build report could contain outputs from BMS, for example: USER1.DBB.COPYBOOKS(ESPMLIS), USER1.DBB.DBRM(EPSCMORT), USER1.DBB.LOAD(EPSCMORT), but the user is only interested in publishing load modules in USER1.DBB.LOAD. The build property 'loadDatasets' should then be set to 'USER1.DBB.LOAD' 
 3. It then invokes CopyToHFS to copy the load modules from the PDSe to a temporary directory on zFS.
 4. It packages these load files into a tar file, and compute the SHA1 and MD5 checksums.