From 05472dfcda37b37b20ed1f79c9fed2c06eb29cba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 11:36:23 +0000 Subject: [PATCH 01/12] Update build script to use updated Common_scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1b74fed..934ffe3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,7 +157,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - private String buildstampprefix + String buildstampprefix int revision String buildstamp From 7323493b05ab01ff46cf1d86207dfc19ff761870 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:28:15 +0000 Subject: [PATCH 02/12] Add non-build startup boilerplate --- autobuilder.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..1210aab 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -56,6 +56,28 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA return meta } +// Start up boilderplate +def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = getmetafolderscript().metareset(script) + meta.productinfo.set("", "", "") + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } From eb07ef6dc046a36c85c16ea32799a756a7eda3b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:43:08 +0000 Subject: [PATCH 03/12] Try to use the REPO_ROOT environment variable --- versioncontrol/git.groovy | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..730335a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,12 +19,8 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" - } else { - reporootpath = "" +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = env.REPO_ROOT.trim(), int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + if (repoRoot == "") { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } checkout([ @@ -39,7 +35,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: repoRoot, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], sparsecheckoutconfig(sparseitemlist) From 31196dbede6b5c339c21dfb24080e928bf4bd435 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:47:04 +0000 Subject: [PATCH 04/12] Revert "Try to use the REPO_ROOT environment variable" This reverts commit aa04eb23b85ac42d236d21dc489f3a03038fbfba. --- versioncontrol/git.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 730335a..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,8 +19,12 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = env.REPO_ROOT.trim(), int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - if (repoRoot == "") { +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } checkout([ @@ -35,7 +39,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: repoRoot, shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], sparsecheckoutconfig(sparseitemlist) From 9f47550d0e051c80f3ef9c44163558dd6b2d5e7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 19:07:43 +0000 Subject: [PATCH 05/12] Try to speed up release repo check out --- autobuilder.groovy | 4 ++-- release/registerrepo.groovy | 4 ++-- versioncontrol/git.groovy | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 1210aab..b9a36d4 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -32,7 +32,7 @@ def getmetafolderscript() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -48,7 +48,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def rr = load("${commonscripts()}/release/registerrepo.groovy") - rr.registerrepo(meta, releaserepo) + rr.registerrepo(meta, releaserepo, repoRootSuffix) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index ff4d85d..2dffdc7 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -2,9 +2,9 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo) { +def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix) } meta.releaseinfo.set(releaserepo) } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..f7d10bb 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,10 +19,11 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" + if (reporootenv != null && repoRootSuffix != "") { + reporootpath = "${reporootenv.trim()}/${repoRootSuffix}" + echo "Using release repo: ${reporootpath}" } else { reporootpath = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" From 643161784419d1c66ab10c56e3db6195c3d55778 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 21:02:16 +0000 Subject: [PATCH 06/12] Update build script to use updated Common_scripts - Installers --- install/installhelper.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 03ee9e4..0447cb8 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -4,7 +4,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { - def artifactKey = installerDetails.get(installer)[0] + def artifactKey = installerDetails.get(installer)[1] (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) @@ -17,7 +17,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { // Export any required binaries to the /bin folder def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { - (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { From 2ab7619f1a649a54e465a5dd8729e3e09947c603 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 21:15:21 +0000 Subject: [PATCH 07/12] Update build script to use updated Common_scripts - Installers --- install/installhelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 0447cb8..819a580 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -19,10 +19,10 @@ def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { - (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { - binariesSourceFolder = "${destinationFolder}/${binaryFolder}" - binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + def binariesSourceFolder = "${destFolder}/${binaryFolder}" + def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) } From 134f5497362b537147699257f3ac79322ca9ff61 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 23:14:18 +0000 Subject: [PATCH 08/12] Update build script to use updated Common_scripts - Installers --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index b9a36d4..63ca2de 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -67,7 +67,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("", "", "") + meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) From 3264655356b72f27aa28e5fbd4ccb5a77a47efd3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 18:20:05 +0000 Subject: [PATCH 09/12] Compare hps_0.h files --- differences/file.groovy | 12 ++++++++++++ differences/json.groovy | 4 +--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 differences/file.groovy diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..f5e93da --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,12 @@ +def comparefiles(importedfile, currentfile) { + if (isUnix()) { + sh """ + diff -c <"${importedfile}" <"${currentfile}" + """ + } + else { + error "Windows not currently supporting file comparison" + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy index 19b7cf6..5ac37f2 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -1,12 +1,10 @@ def comparefiles(importedjsonfile, currentjsonfile) { - env.IMPORTED_JSON="${importedjsonfile}" - env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ which jq if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output - diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") """ } else { From 2013c582c73ae5adf5b1c0bc26da48963faac83a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 19:01:59 +0000 Subject: [PATCH 10/12] Compare hps_0.h files --- differences/file.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/file.groovy b/differences/file.groovy index f5e93da..dee78b9 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,7 +1,7 @@ def comparefiles(importedfile, currentfile) { if (isUnix()) { sh """ - diff -c <"${importedfile}" <"${currentfile}" + diff -c "${importedfile}" "${currentfile}" """ } else { From 06c096fc7c054611d8847394e52bf1916163633b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 19:17:49 +0000 Subject: [PATCH 11/12] Compare hps_0.h files --- differences/file.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/file.groovy b/differences/file.groovy index dee78b9..ab4102f 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,7 +1,11 @@ -def comparefiles(importedfile, currentfile) { +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { if (isUnix()) { + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } sh """ - diff -c "${importedfile}" "${currentfile}" + diff -c ${wsparameter} "${importedfile}" "${currentfile}" """ } else { From f61e44d6dbe4ddfb39935d26dd14eb6edaf7a251 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 15 Feb 2019 13:42:58 +0000 Subject: [PATCH 12/12] Debug publishable list addition --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 934ffe3..52ceb06 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -486,6 +486,7 @@ class MetaInfo extends GISJsonSerialised { void addPublishable(String filepath) { def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") publishableinfo.add(pi) }