From a80034e809b74e60c5109b7349c8e4af7a53e279 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 11:18:49 +0000 Subject: [PATCH 001/110] Certain components are built from the same source repository so therefore their version numbers should not be incremented when they are imported into a build component using the same source repository --- artifacts/metaexport.groovy | 5 +++-- import/importer.groovy | 12 ++++++++---- install/installhelper.groovy | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0ca5da7..043c4de 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -9,11 +9,12 @@ def combineSubjobArtifacts(meta, subJobs) { if (subJobObject != null) { def subJobName = subJobObject.getProjectName() def subJobNumber = subJobObject.getNumber() - destFolder = subJob[1] + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) artifactFolders.add("${destFolder}/**") } } diff --git a/import/importer.groovy b/import/importer.groovy index 1f89426..5c996b4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,6 +1,6 @@ versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -10,11 +10,13 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -27,7 +29,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 093ddef..03ee9e4 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -5,9 +5,9 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] - (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) artifactFetched.put(artifactKey, importmeta) } } From 18272f5bc97fedf92d6bf70bc23d15d96e778bcf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 22:14:31 +0000 Subject: [PATCH 002/110] Alter location of differences file so that it no longer gets published --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2ef8e22..461a8fb 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -419,6 +419,7 @@ class PublishableInfo extends GISJsonSerialised { class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" + private static String DIFFFOLDERNAME = "diff" private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" @@ -565,7 +566,7 @@ class MetaInfo extends GISJsonSerialised { } static String difffilepath() { - return METAFOLDERNAME + "/" + METADIFFFILENAME + return DIFFFOLDERNAME + "/" + METADIFFFILENAME } MetaInfo importrelated(String destpath) { From a4fc87477f8bb4a9cfaf62feff4b6b14fc281897 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 23:07:18 +0000 Subject: [PATCH 003/110] Alter location of differences file so that it no longer gets published --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 461a8fb..efc8df2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -423,6 +423,7 @@ class MetaInfo extends GISJsonSerialised { private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String DIFFARTIFACTS = "${DIFFFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -621,7 +622,7 @@ class MetaInfo extends GISJsonSerialised { ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { - def String artifactlist = METAARTIFACTS + def String artifactlist = "${METAARTIFACTS},${DIFFARTIFACTS}" releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { artifactlist += "," + releaseitem From 54185b50486c7eef935aabbe62e77f14cabe2dfc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 31 Jan 2019 18:32:44 +0000 Subject: [PATCH 004/110] Select the artifacts from a specific build for the duration of the job --- meta/folder.groovy | 4 ++-- release/do.groovy | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index efc8df2..1b74fed 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -118,8 +118,8 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { - private String buildjobname - private int buildnumber + String buildjobname + int buildnumber private String buildmachine private String builddate diff --git a/release/do.groovy b/release/do.groovy index 1a785b3..1d9e721 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -92,7 +92,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { } } -def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { @@ -104,10 +104,10 @@ def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { echo "Copying artifacts: ${filterpath} to ${tempfolder}" copyArtifacts( filter: filterpath, - projectName: BUILD_JOB, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR + parameterName: " ${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}") ], target: tempfolder ) @@ -315,7 +315,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } -def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { +def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { def buildmeta = buildinfo.meta def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" @@ -327,7 +327,7 @@ def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp) gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) @@ -368,12 +368,12 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] if (toplevelid != key) { - releasecomponent(buildinfo, otherreleasetags, dryrunpush) + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) } } // Do the top level last def toplevelbuildinfo = componentReleaseInfo[toplevelid] - releasecomponent(toplevelbuildinfo, otherreleasetags, dryrunpush) + releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) return [summarystring, markunstable, unstablereason] } From 4b6c3aff23174460d121db08793cb9a586862976 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 31 Jan 2019 18:38:21 +0000 Subject: [PATCH 005/110] Select the artifacts from a specific build for the duration of the job --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1d9e721..0f65035 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -107,7 +107,7 @@ def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, Strin projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: " ${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}") + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" ], target: tempfolder ) From 628fe40745113bb353cb60021fc0f61ce527321e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 31 Jan 2019 22:35:18 +0000 Subject: [PATCH 006/110] Clone depth 1 is causing issues during release to git --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 0f65035..604581a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -57,7 +57,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + sh("git clone ${releaserepo} ${checkoutfolder}") } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { From 05472dfcda37b37b20ed1f79c9fed2c06eb29cba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 11:36:23 +0000 Subject: [PATCH 007/110] Update build script to use updated Common_scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1b74fed..934ffe3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,7 +157,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - private String buildstampprefix + String buildstampprefix int revision String buildstamp From 7323493b05ab01ff46cf1d86207dfc19ff761870 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:28:15 +0000 Subject: [PATCH 008/110] Add non-build startup boilerplate --- autobuilder.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..1210aab 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -56,6 +56,28 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA return meta } +// Start up boilderplate +def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = getmetafolderscript().metareset(script) + meta.productinfo.set("", "", "") + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } From eb07ef6dc046a36c85c16ea32799a756a7eda3b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:43:08 +0000 Subject: [PATCH 009/110] Try to use the REPO_ROOT environment variable --- versioncontrol/git.groovy | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..730335a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,12 +19,8 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" - } else { - reporootpath = "" +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = env.REPO_ROOT.trim(), int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + if (repoRoot == "") { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } checkout([ @@ -39,7 +35,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: repoRoot, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], sparsecheckoutconfig(sparseitemlist) From 31196dbede6b5c339c21dfb24080e928bf4bd435 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:47:04 +0000 Subject: [PATCH 010/110] Revert "Try to use the REPO_ROOT environment variable" This reverts commit aa04eb23b85ac42d236d21dc489f3a03038fbfba. --- versioncontrol/git.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 730335a..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,8 +19,12 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = env.REPO_ROOT.trim(), int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - if (repoRoot == "") { +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } checkout([ @@ -35,7 +39,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: repoRoot, shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], sparsecheckoutconfig(sparseitemlist) From 9f47550d0e051c80f3ef9c44163558dd6b2d5e7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 19:07:43 +0000 Subject: [PATCH 011/110] Try to speed up release repo check out --- autobuilder.groovy | 4 ++-- release/registerrepo.groovy | 4 ++-- versioncontrol/git.groovy | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 1210aab..b9a36d4 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -32,7 +32,7 @@ def getmetafolderscript() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -48,7 +48,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def rr = load("${commonscripts()}/release/registerrepo.groovy") - rr.registerrepo(meta, releaserepo) + rr.registerrepo(meta, releaserepo, repoRootSuffix) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index ff4d85d..2dffdc7 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -2,9 +2,9 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo) { +def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix) } meta.releaseinfo.set(releaserepo) } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..f7d10bb 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,10 +19,11 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" + if (reporootenv != null && repoRootSuffix != "") { + reporootpath = "${reporootenv.trim()}/${repoRootSuffix}" + echo "Using release repo: ${reporootpath}" } else { reporootpath = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" From 643161784419d1c66ab10c56e3db6195c3d55778 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 21:02:16 +0000 Subject: [PATCH 012/110] Update build script to use updated Common_scripts - Installers --- install/installhelper.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 03ee9e4..0447cb8 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -4,7 +4,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { - def artifactKey = installerDetails.get(installer)[0] + def artifactKey = installerDetails.get(installer)[1] (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) @@ -17,7 +17,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { // Export any required binaries to the /bin folder def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { - (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { From 2ab7619f1a649a54e465a5dd8729e3e09947c603 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 21:15:21 +0000 Subject: [PATCH 013/110] Update build script to use updated Common_scripts - Installers --- install/installhelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 0447cb8..819a580 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -19,10 +19,10 @@ def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { - (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { - binariesSourceFolder = "${destinationFolder}/${binaryFolder}" - binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + def binariesSourceFolder = "${destFolder}/${binaryFolder}" + def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) } From 134f5497362b537147699257f3ac79322ca9ff61 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 23:14:18 +0000 Subject: [PATCH 014/110] Update build script to use updated Common_scripts - Installers --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index b9a36d4..63ca2de 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -67,7 +67,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("", "", "") + meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) From 3264655356b72f27aa28e5fbd4ccb5a77a47efd3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 18:20:05 +0000 Subject: [PATCH 015/110] Compare hps_0.h files --- differences/file.groovy | 12 ++++++++++++ differences/json.groovy | 4 +--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 differences/file.groovy diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..f5e93da --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,12 @@ +def comparefiles(importedfile, currentfile) { + if (isUnix()) { + sh """ + diff -c <"${importedfile}" <"${currentfile}" + """ + } + else { + error "Windows not currently supporting file comparison" + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy index 19b7cf6..5ac37f2 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -1,12 +1,10 @@ def comparefiles(importedjsonfile, currentjsonfile) { - env.IMPORTED_JSON="${importedjsonfile}" - env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ which jq if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output - diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") """ } else { From 2013c582c73ae5adf5b1c0bc26da48963faac83a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 19:01:59 +0000 Subject: [PATCH 016/110] Compare hps_0.h files --- differences/file.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/file.groovy b/differences/file.groovy index f5e93da..dee78b9 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,7 +1,7 @@ def comparefiles(importedfile, currentfile) { if (isUnix()) { sh """ - diff -c <"${importedfile}" <"${currentfile}" + diff -c "${importedfile}" "${currentfile}" """ } else { From 06c096fc7c054611d8847394e52bf1916163633b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 19:17:49 +0000 Subject: [PATCH 017/110] Compare hps_0.h files --- differences/file.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/file.groovy b/differences/file.groovy index dee78b9..ab4102f 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,7 +1,11 @@ -def comparefiles(importedfile, currentfile) { +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { if (isUnix()) { + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } sh """ - diff -c "${importedfile}" "${currentfile}" + diff -c ${wsparameter} "${importedfile}" "${currentfile}" """ } else { From f61e44d6dbe4ddfb39935d26dd14eb6edaf7a251 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 15 Feb 2019 13:42:58 +0000 Subject: [PATCH 018/110] Debug publishable list addition --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 934ffe3..52ceb06 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -486,6 +486,7 @@ class MetaInfo extends GISJsonSerialised { void addPublishable(String filepath) { def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") publishableinfo.add(pi) } From 0f636ea4276e66e4656eaf8afb2e9ae760abba27 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 16:07:52 +0000 Subject: [PATCH 019/110] Try to create WORKSPACE relative paths --- meta/folder.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 52ceb06..245d961 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -554,13 +554,16 @@ class MetaInfo extends GISJsonSerialised { } String relativize(String path) { - def Path thispath = Paths.get(path) - if (thispath.startsWith(script.WORKSPACE)) { + def Path mypath = Paths.get(path) + String pathRelative + if (mypath.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(thispath) - return wsrelativePath.toString() + def Path wsrelativePath = wspath.relativize(mypath) + pathRelative = wsrelativePath.toString() + } else { + pathRelative = mypath.toString() } - return path + return pathRelative } static String metafilepath() { From dc6e7c6b1d5a0e54aec98ce61539ae27b855c2dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:05:25 +0000 Subject: [PATCH 020/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 245d961..d7cbc4f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -556,12 +556,15 @@ class MetaInfo extends GISJsonSerialised { String relativize(String path) { def Path mypath = Paths.get(path) String pathRelative - if (mypath.startsWith(script.WORKSPACE)) { + script.echo("Seeing if ${path} begins with from ${script.WORKSPACE}") + if (path.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) pathRelative = wsrelativePath.toString() + script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}" } else { pathRelative = mypath.toString() + script.echo("Path relative is unchanged: ${pathRelative}" } return pathRelative } From de9e4d82ed9664de567db925c1213be31cc8e84a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:07:33 +0000 Subject: [PATCH 021/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d7cbc4f..0c94a12 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -561,7 +561,7 @@ class MetaInfo extends GISJsonSerialised { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) pathRelative = wsrelativePath.toString() - script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}" + script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") } else { pathRelative = mypath.toString() script.echo("Path relative is unchanged: ${pathRelative}" From 6ca84a13fecedfda50a8a8f594bf761b60a26fc9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:18:29 +0000 Subject: [PATCH 022/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0c94a12..f6797a2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,7 +564,7 @@ class MetaInfo extends GISJsonSerialised { script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") } else { pathRelative = mypath.toString() - script.echo("Path relative is unchanged: ${pathRelative}" + script.echo("Path relative is unchanged: ${pathRelative}") } return pathRelative } From d66b7b80d602550ccae4aca1b1483f36f19b4a67 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:21:14 +0000 Subject: [PATCH 023/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f6797a2..ce041d0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -556,7 +556,7 @@ class MetaInfo extends GISJsonSerialised { String relativize(String path) { def Path mypath = Paths.get(path) String pathRelative - script.echo("Seeing if ${path} begins with from ${script.WORKSPACE}") + script.echo("Seeing if ${path} begins with ${script.WORKSPACE}") if (path.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) From 736f0ece675736032cc8b1dc1f33c313015648e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:45:39 +0000 Subject: [PATCH 024/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ce041d0..74c00cc 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -553,17 +553,14 @@ class MetaInfo extends GISJsonSerialised { deserialiseJson(rawmeta) } - String relativize(String path) { - def Path mypath = Paths.get(path) + String relativize(String mypath) { String pathRelative - script.echo("Seeing if ${path} begins with ${script.WORKSPACE}") - if (path.startsWith(script.WORKSPACE)) { - def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(mypath) - pathRelative = wsrelativePath.toString() - script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") } else { - pathRelative = mypath.toString() + pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } return pathRelative From d2bd59bfa5c3a6d922a118e86fc46e707f4b5f90 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 18:07:16 +0000 Subject: [PATCH 025/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 74c00cc..1f90c61 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -563,6 +563,13 @@ class MetaInfo extends GISJsonSerialised { pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } + // Strip leading / if any + pathRelative = pathRelative.replaceAll("^/+", "") + // Strip leading \ if any + pathRelative = pathRelative.replaceAll("^\\+", "") + + script.echo("Path relative is ${pathRelative}") + return pathRelative } From 4c7726123e43370c5c6397b678615717ec11e8ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:06:38 +0000 Subject: [PATCH 026/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1f90c61..3a4d464 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,9 +564,11 @@ class MetaInfo extends GISJsonSerialised { script.echo("Path relative is unchanged: ${pathRelative}") } // Strip leading / if any - pathRelative = pathRelative.replaceAll("^/+", "") + pathRelative = pathRelative.replaceAll('^/+', '') // Strip leading \ if any - pathRelative = pathRelative.replaceAll("^\\+", "") + pathRelative = pathRelative.replaceAll('^\\+', '') + // Convert \ with / + pathRelative = pathRelative.replaceAll('\\', '/') script.echo("Path relative is ${pathRelative}") From da35fd3aa07e1ee84822752e74a9b2fcc3a5ec3e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:23:02 +0000 Subject: [PATCH 027/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 3a4d464..adfee97 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -568,7 +568,7 @@ class MetaInfo extends GISJsonSerialised { // Strip leading \ if any pathRelative = pathRelative.replaceAll('^\\+', '') // Convert \ with / - pathRelative = pathRelative.replaceAll('\\', '/') + pathRelative = pathRelative.replace('\\', '/') script.echo("Path relative is ${pathRelative}") From 06ca80c596d36ce9c758edd6982132eec3e3c44a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:46:23 +0000 Subject: [PATCH 028/110] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index adfee97..f9d4bf3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -563,12 +563,10 @@ class MetaInfo extends GISJsonSerialised { pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') // Strip leading / if any pathRelative = pathRelative.replaceAll('^/+', '') - // Strip leading \ if any - pathRelative = pathRelative.replaceAll('^\\+', '') - // Convert \ with / - pathRelative = pathRelative.replace('\\', '/') script.echo("Path relative is ${pathRelative}") From a7da6556d91007e844760c552e804cce87f312ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 00:01:14 +0000 Subject: [PATCH 029/110] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 42 ++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..d333af3 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -28,22 +28,50 @@ def doKillLeftovers() { } } +def doCheck(checkpathlist, build_variant = "") { + def result = 0 + for (checkpath in checkpathlist) { + echo "Checking file at: ${checkpath}" + script { + echo "Build variant is ${build_variant}" + if (build_variant) { + def srcFile = "${WORKSPACE}/${checkpath}" + def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + result = sh( script: """ + // Print out dependencies of the built file + readelf -d "${srcFileExpanded}" | grep NEEDED + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + if (result != 0) { + return result + } + } + } + } + return result +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 + def destFolder = "" + def srcFile = "" + def srcFileExpanded = "" for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + destFolder = "${exportfolder}/${build_variant}" + srcFile = "${WORKSPACE}/${exportpath}" + srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" - env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + destFolder = "${exportfolder}/${project}" + srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { @@ -162,6 +190,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } + result = doCheck(exportpathlist, build_variant) + if (result != 0) { + error "Failed to check files...." + } result = doExport(exportfolder, exportpathlist, "", build_variant) if (result != 0) { error "Failed to copy...." From 4c7053017eadcc4d6fe308bf6abbe087de31f0dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 00:33:36 +0000 Subject: [PATCH 030/110] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index d333af3..2dbd6ea 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -74,7 +74,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { srcFileExpanded = "${WORKSPACE}/${exportpath}" } } - dir (env.destFolder) { + dir (destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { result = sh( script: """ @@ -88,10 +88,10 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } } else { - result = bat( script: ''' - copy \"%srcFileExpanded%\" \"%destFolder%\" + result = bat( script: """ + copy "${srcFileExpanded}" "${destFolder}" if errorlevel 1 EXIT /b 1 - ''', + """, returnStatus: true ) if (result != 0) { From 49f1b99e477c945156e9caaa4215ff8659a360eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 07:55:15 +0000 Subject: [PATCH 031/110] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 2dbd6ea..302b4f8 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -37,8 +37,9 @@ def doCheck(checkpathlist, build_variant = "") { if (build_variant) { def srcFile = "${WORKSPACE}/${checkpath}" def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + // Print out dependencies of the built file + echo "Showing dependencies of ${srcFileExpanded}" result = sh( script: """ - // Print out dependencies of the built file readelf -d "${srcFileExpanded}" | grep NEEDED if [ \$? -ne 0 ]; then exit 1; fi """, From c4247d45ed5bfcfeab751fdb6595158f51c14bdc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 18:37:55 +0000 Subject: [PATCH 032/110] Add publish item selector script --- publish/publish_item_select.groovy | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 publish/publish_item_select.groovy diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy new file mode 100755 index 0000000..747b9f6 --- /dev/null +++ b/publish/publish_item_select.groovy @@ -0,0 +1,90 @@ +#!/usr/local/bin/groovy +metarepo="git@hydra:jenkinsautobuild/Common_scripts.git" + +import hudson.model.* + +def log(message) { + boolean dolog = true + if (dolog) { + println message + } +} + +println "hello" + +def createfolder(String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + log("Creating directory: ${folder}") + def s = ['mkdir', '-p', folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def gitCloneOrPull(String repo, String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + if (!new File(folder).exists()) { + log("Cloning: ${repo} to: ${folder}") + createfolder(folder) + def s = ['git', 'clone', repo, folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } else { + log("Pull in: ${folder}") + def s = ['/bin/sh', '-c', "cd ${folder}"].execute() | ['git', 'pull'].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def String makeworking() { + def r = ['whoami'].execute() + r.waitFor() + def user = r.text.trim() + + log("User is ${user}") + def rootfolder = "/home/${user}/tmp" + createfolder(rootfolder) + + def commonscripts = "${rootfolder}/commonscripts" + gitCloneOrPull(metarepo, commonscripts); + +} + +def List getItems() { + def List items = new ArrayList() + try { + log("Getting items") + def p = ['pwd'].execute() + p.waitFor() + items.add(p.text) + items.add(metarepo) + items.add(metalevels) + + items.add("ATLAS_SERVICES CORE_INSTALL installers/**") + + items.add("Item 2") + items.add("Item 3") + return items + } catch (Exception e) { + items.add("Error ${e}") + return items + } +} + +makeworking() +count = 0 +for (item in getItems()) { + log (count) + count += 1 + log (item) +} \ No newline at end of file From 07c3e7ce16f61f387e878f610c76da079bf79ae0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 18:51:12 +0000 Subject: [PATCH 033/110] Add publish item selector script --- publish/publish_item_select.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 747b9f6..50090a8 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -29,7 +29,7 @@ def gitCloneOrPull(String repo, String folder) { def sout = new StringBuilder() def serr = new StringBuilder() - if (!new File(folder).exists()) { + if (!(new File(folder).exists())) { log("Cloning: ${repo} to: ${folder}") createfolder(folder) def s = ['git', 'clone', repo, folder].execute() @@ -37,7 +37,7 @@ def gitCloneOrPull(String repo, String folder) { s.waitFor() } else { log("Pull in: ${folder}") - def s = ['/bin/sh', '-c', "cd ${folder}"].execute() | ['git', 'pull'].execute() + def s = ["/bin/sh -c cd ${folder}; git pull"].execute() s.consumeProcessOutput(sout, serr) s.waitFor() } From fdff1c1ef7f568673752e35d1db2c138d377a9f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 18:55:44 +0000 Subject: [PATCH 034/110] Add publish item selector script --- publish/publish_item_select.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 50090a8..68215ff 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -37,7 +37,7 @@ def gitCloneOrPull(String repo, String folder) { s.waitFor() } else { log("Pull in: ${folder}") - def s = ["/bin/sh -c cd ${folder}; git pull"].execute() + def s = ["/bin/bash -c cd ${folder}; pwd; git pull"].execute() s.consumeProcessOutput(sout, serr) s.waitFor() } From d7a8dd14add18cd947df7e4dac99331efe898172 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 19:28:47 +0000 Subject: [PATCH 035/110] Add publish item selector script --- publish/publish_item_select.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 68215ff..56f7e37 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -37,7 +37,7 @@ def gitCloneOrPull(String repo, String folder) { s.waitFor() } else { log("Pull in: ${folder}") - def s = ["/bin/bash -c cd ${folder}; pwd; git pull"].execute() + def s = ['git', 'pull'].execute(null, new File(folder)) s.consumeProcessOutput(sout, serr) s.waitFor() } From 6cc9f1928d2dc28693bbc10f1165b80a3724b688 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 23:34:08 +0000 Subject: [PATCH 036/110] Add publish item selector script --- publish/publish_item_select.groovy | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 56f7e37..ef39d78 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,5 +1,7 @@ #!/usr/local/bin/groovy -metarepo="git@hydra:jenkinsautobuild/Common_scripts.git" +metarepoext = "git@hydra:HMB_XR_5601_Releases/hmb_xr_5601_whole.git" + +commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" import hudson.model.* @@ -46,6 +48,15 @@ def gitCloneOrPull(String repo, String folder) { return [sout, serr] } +def loadmetaclass(String commonscripts) { + def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" + log("Loading script: ${metafolderscriptfile}") + def script = new GroovyScriptEngine('.').with { + loadScriptByName(metafolderscriptfile) + } + return script +} + def String makeworking() { def r = ['whoami'].execute() r.waitFor() @@ -56,8 +67,10 @@ def String makeworking() { createfolder(rootfolder) def commonscripts = "${rootfolder}/commonscripts" - gitCloneOrPull(metarepo, commonscripts); + gitCloneOrPull(commonscriptsrepo, commonscripts); + def metascript = loadmetaclass(commonscripts) + return metascript } def List getItems() { @@ -67,7 +80,7 @@ def List getItems() { def p = ['pwd'].execute() p.waitFor() items.add(p.text) - items.add(metarepo) + items.add(metarepoext) items.add(metalevels) items.add("ATLAS_SERVICES CORE_INSTALL installers/**") @@ -82,6 +95,7 @@ def List getItems() { } makeworking() +loadmetaclass() count = 0 for (item in getItems()) { log (count) From 39d54ecfa090ffa7d6f69a97c4b5ca66ccacfca2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 11:17:25 +0000 Subject: [PATCH 037/110] Add publish item selector script --- meta/folder.groovy | 15 +++++++++- publish/publish_item_select.groovy | 48 ++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f9d4bf3..9f747ad 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ import java.nio.file.Paths import java.nio.file.Path +import groovy.json.JsonSlurperClassic abstract class GISJsonSerialised implements Serializable { def script @@ -552,7 +553,13 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } - + + def deserialiseJsonSlurper(String filepath) { + def jsonfile = readFile(file: filepath) + def rawmeta = new JsonSlurperClassic().parseText(jsonfile) + deserialiseJson(rawmeta) + } + String relativize(String mypath) { String pathRelative script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") @@ -726,4 +733,10 @@ MetaInfo metafromfolder(String folder = "") { return buildmeta } +MetaInfo metafromfolderSlurper(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + return this; diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index ef39d78..7eeae29 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,5 +1,7 @@ #!/usr/local/bin/groovy +jobname="anything" metarepoext = "git@hydra:HMB_XR_5601_Releases/hmb_xr_5601_whole.git" +metalevels = 2 commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" @@ -48,13 +50,25 @@ def gitCloneOrPull(String repo, String folder) { return [sout, serr] } -def loadmetaclass(String commonscripts) { +def loadmetaclass(String commonscripts, String metafolder) { def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" log("Loading script: ${metafolderscriptfile}") - def script = new GroovyScriptEngine('.').with { - loadScriptByName(metafolderscriptfile) - } - return script + + GroovyShell shell = new GroovyShell() + def metafolderscript = shell.parse(new File(metafolderscriptfile)) + log("metafolderscript is ${metafolderscript.getClass()}") + def meta = metafolderscript.metafromfolder(metafolder) + log("meta is ${meta.getClass()}") + + + + //Class groovyClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(metafolderscriptfile) + //log("groovyClass is ${groovyClass.getClass()}") + //GroovyObject metafolderobject = (GroovyObject) groovyClass.newInstance() + //log("metafolderobject is ${metafolderobject.getClass()}") + //metaObject = new metafolderobject.MetaInfo(this) + //log("metaobject is ${metaObject.getClass()}") + return meta } def String makeworking() { @@ -67,10 +81,19 @@ def String makeworking() { createfolder(rootfolder) def commonscripts = "${rootfolder}/commonscripts" - gitCloneOrPull(commonscriptsrepo, commonscripts); + gitCloneOrPull(commonscriptsrepo, commonscripts) - def metascript = loadmetaclass(commonscripts) - return metascript + // Clone or pull the relevant meta repo + def metareporoot = "${rootfolder}/${jobname}" + gitCloneOrPull(metarepoext, metareporoot) + def metafolder = "${metareporoot}/meta" + + // Load the meta folder class that interprets the meta repo + def metascript = loadmetaclass(commonscripts, metafolder) + + // Read in the meta + meta = metascript.metafromfolder(metafolder) + return meta } def List getItems() { @@ -79,7 +102,7 @@ def List getItems() { log("Getting items") def p = ['pwd'].execute() p.waitFor() - items.add(p.text) + items.add(p.text.trim()) items.add(metarepoext) items.add(metalevels) @@ -95,10 +118,9 @@ def List getItems() { } makeworking() -loadmetaclass() count = 0 -for (item in getItems()) { - log (count) +def items = getItems() +for (item in items) { count += 1 - log (item) + log ("${count}: ${item}") } \ No newline at end of file From 6746c4dccf1ca52ad77424e1917a989880107542 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 11:24:26 +0000 Subject: [PATCH 038/110] Add publish item selector script --- meta/folder.groovy | 5 +++-- publish/publish_item_select.groovy | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9f747ad..d751f5e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -555,8 +555,9 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJsonSlurper(String filepath) { - def jsonfile = readFile(file: filepath) - def rawmeta = new JsonSlurperClassic().parseText(jsonfile) + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) deserialiseJson(rawmeta) } diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 7eeae29..4af7c2b 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -57,7 +57,7 @@ def loadmetaclass(String commonscripts, String metafolder) { GroovyShell shell = new GroovyShell() def metafolderscript = shell.parse(new File(metafolderscriptfile)) log("metafolderscript is ${metafolderscript.getClass()}") - def meta = metafolderscript.metafromfolder(metafolder) + def meta = metafolderscript.metafromfolderSlurper(metafolder) log("meta is ${meta.getClass()}") From 875e21b092e721f5e35183da36a11832d57c18da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 11:53:57 +0000 Subject: [PATCH 039/110] Add publish item selector script --- publish/publish_item_select.groovy | 53 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 4af7c2b..f02b800 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -60,18 +60,10 @@ def loadmetaclass(String commonscripts, String metafolder) { def meta = metafolderscript.metafromfolderSlurper(metafolder) log("meta is ${meta.getClass()}") - - - //Class groovyClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(metafolderscriptfile) - //log("groovyClass is ${groovyClass.getClass()}") - //GroovyObject metafolderobject = (GroovyObject) groovyClass.newInstance() - //log("metafolderobject is ${metafolderobject.getClass()}") - //metaObject = new metafolderobject.MetaInfo(this) - //log("metaobject is ${metaObject.getClass()}") return meta } -def String makeworking() { +def makeworking() { def r = ['whoami'].execute() r.waitFor() def user = r.text.trim() @@ -89,37 +81,46 @@ def String makeworking() { def metafolder = "${metareporoot}/meta" // Load the meta folder class that interprets the meta repo - def metascript = loadmetaclass(commonscripts, metafolder) - - // Read in the meta - meta = metascript.metafromfolder(metafolder) + def meta = loadmetaclass(commonscripts, metareporoot) + log("2: meta is ${meta.getClass()}") + return meta } -def List getItems() { - def List items = new ArrayList() +def List getItems(meta) { + def List menuitems = new ArrayList() + def metaitems = meta.releasewalkerunique(metalevels) + log("metaitems is ${metaitems.getClass()}") + try { log("Getting items") def p = ['pwd'].execute() p.waitFor() - items.add(p.text.trim()) - items.add(metarepoext) - items.add(metalevels) + menuitems.add(p.text.trim()) + menuitems.add(metarepoext) + menuitems.add(metalevels) - items.add("ATLAS_SERVICES CORE_INSTALL installers/**") + menuitems.add("ATLAS_SERVICES CORE_INSTALL installers/**") - items.add("Item 2") - items.add("Item 3") - return items + menuitems.add("Item 2") + menuitems.add("Item 3") + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + menuitems.add(releaseinfo.offsetpath) + //MetaInfo meta + } + return menuitems } catch (Exception e) { - items.add("Error ${e}") - return items + menuitems.add("Error ${e}") + return menuitems } } -makeworking() +def meta = makeworking() +log("3: meta is ${meta.getClass()}") + count = 0 -def items = getItems() +def items = getItems(meta) for (item in items) { count += 1 log ("${count}: ${item}") From b480d4a965d99e1463a7f96046b36bcf7491facf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 12:40:37 +0000 Subject: [PATCH 040/110] Add publish item selector script --- publish/publish_item_select.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index f02b800..47b64dc 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,6 +1,6 @@ #!/usr/local/bin/groovy -jobname="anything" -metarepoext = "git@hydra:HMB_XR_5601_Releases/hmb_xr_5601_whole.git" +jobname="anythingnew" +metarepoext = "git@hydra:print-server-releases/print_server_whole.git" metalevels = 2 commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" @@ -124,4 +124,6 @@ def items = getItems(meta) for (item in items) { count += 1 log ("${count}: ${item}") -} \ No newline at end of file +} + +return items \ No newline at end of file From 18f6b42a56f11aee83ea1d62b0a8d013d70782fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 3 Mar 2019 18:02:51 +0000 Subject: [PATCH 041/110] Add publish item selector script; Add eclipse groovy project --- publish/.classpath | 9 +++++++ publish/.gitignore | 7 ++++++ publish/.project | 18 ++++++++++++++ .../org.eclipse.jdt.groovy.core.prefs | 3 +++ publish/publish_item_select.groovy | 24 +------------------ 5 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 publish/.classpath create mode 100644 publish/.gitignore create mode 100644 publish/.project create mode 100644 publish/.settings/org.eclipse.jdt.groovy.core.prefs diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 47b64dc..a7cfa1b 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -5,8 +5,6 @@ metalevels = 2 commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" -import hudson.model.* - def log(message) { boolean dolog = true if (dolog) { @@ -52,14 +50,10 @@ def gitCloneOrPull(String repo, String folder) { def loadmetaclass(String commonscripts, String metafolder) { def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" - log("Loading script: ${metafolderscriptfile}") GroovyShell shell = new GroovyShell() def metafolderscript = shell.parse(new File(metafolderscriptfile)) - log("metafolderscript is ${metafolderscript.getClass()}") def meta = metafolderscript.metafromfolderSlurper(metafolder) - log("meta is ${meta.getClass()}") - return meta } @@ -68,7 +62,6 @@ def makeworking() { r.waitFor() def user = r.text.trim() - log("User is ${user}") def rootfolder = "/home/${user}/tmp" createfolder(rootfolder) @@ -82,8 +75,6 @@ def makeworking() { // Load the meta folder class that interprets the meta repo def meta = loadmetaclass(commonscripts, metareporoot) - log("2: meta is ${meta.getClass()}") - return meta } @@ -93,21 +84,9 @@ def List getItems(meta) { log("metaitems is ${metaitems.getClass()}") try { - log("Getting items") - def p = ['pwd'].execute() - p.waitFor() - menuitems.add(p.text.trim()) - menuitems.add(metarepoext) - menuitems.add(metalevels) - - menuitems.add("ATLAS_SERVICES CORE_INSTALL installers/**") - - menuitems.add("Item 2") - menuitems.add("Item 3") for (key in metaitems.keySet()) { def releaseinfo = metaitems[key] menuitems.add(releaseinfo.offsetpath) - //MetaInfo meta } return menuitems } catch (Exception e) { @@ -117,10 +96,9 @@ def List getItems(meta) { } def meta = makeworking() -log("3: meta is ${meta.getClass()}") +def items = getItems(meta) count = 0 -def items = getItems(meta) for (item in items) { count += 1 log ("${count}: ${item}") From c4a5e7a36177d7218b0af68c618bb9122516360c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 3 Mar 2019 22:24:58 +0000 Subject: [PATCH 042/110] Add publish item selector script; Add eclipse groovy project --- publish/publish_item_select.groovy | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index a7cfa1b..2299f37 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,19 +1,18 @@ #!/usr/local/bin/groovy jobname="anythingnew" metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +outputselect = "description" metalevels = 2 +dolog = false commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" def log(message) { - boolean dolog = true if (dolog) { println message } } -println "hello" - def createfolder(String folder) { def sout = new StringBuilder() def serr = new StringBuilder() @@ -78,6 +77,22 @@ def makeworking() { return meta } +def String outputDescription(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } +} + +def String outputSelection(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } +} + def List getItems(meta) { def List menuitems = new ArrayList() def metaitems = meta.releasewalkerunique(metalevels) @@ -85,8 +100,13 @@ def List getItems(meta) { try { for (key in metaitems.keySet()) { - def releaseinfo = metaitems[key] - menuitems.add(releaseinfo.offsetpath) + def releaseinfo = metaitems[key] + if (outputselect == "description") { + outputDescription(menuitems, releaseinfo) + } + if (outputselect == "selection") { + outputSelection(menuitems, releaseinfo) + } } return menuitems } catch (Exception e) { From 2426465d7db8aab7801a86e1e6c1bbd105baabc7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 3 Mar 2019 22:57:57 +0000 Subject: [PATCH 043/110] Add publish item selector script --- publish/publish_item_select.groovy | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 2299f37..99e424f 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,8 +1,16 @@ #!/usr/local/bin/groovy -jobname="anythingnew" -metarepoext = "git@hydra:print-server-releases/print_server_whole.git" -outputselect = "description" -metalevels = 2 + +jobnameext = jobname ?: "jobunique" +metarepoext = metarepo ?: "git@hydra:print-server-releases/print_server_whole.git" +outputselectext = outputselect ?: "description" +metalevelsstringext = metalevels ?: "2" + +//jobnameext = "jobunique" +//metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +//outputselectext = "description" +//metalevelsstringext = "2" + +metalevelsext = metalevelsstringext.toInteger() dolog = false commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" @@ -68,7 +76,7 @@ def makeworking() { gitCloneOrPull(commonscriptsrepo, commonscripts) // Clone or pull the relevant meta repo - def metareporoot = "${rootfolder}/${jobname}" + def metareporoot = "${rootfolder}/${jobnameext}" gitCloneOrPull(metarepoext, metareporoot) def metafolder = "${metareporoot}/meta" @@ -95,16 +103,16 @@ def String outputSelection(menuitems, releaseinfo) { def List getItems(meta) { def List menuitems = new ArrayList() - def metaitems = meta.releasewalkerunique(metalevels) + def metaitems = meta.releasewalkerunique(metalevelsext) log("metaitems is ${metaitems.getClass()}") try { for (key in metaitems.keySet()) { def releaseinfo = metaitems[key] - if (outputselect == "description") { + if (outputselectext == "description") { outputDescription(menuitems, releaseinfo) } - if (outputselect == "selection") { + if (outputselectext == "selection") { outputSelection(menuitems, releaseinfo) } } From bb65e72373fe39938ef6f96b13128db3c53b7e77 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Mar 2019 18:57:10 +0000 Subject: [PATCH 044/110] Clone jobs --- versionnumbering/generate.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 4761174..f4f1bdc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -56,7 +56,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // Handle: // origin/master // origin/2.0.x -def getVersionPrefix(String sourcebranch) { +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -121,6 +121,9 @@ def getVersionPrefix(String sourcebranch) { versionprefix += "0." } } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } echo "Determined version prefix to be: ${versionprefix}" return versionprefix } From 79038df4d3eae65cbfa51ed46a0e82692e4e75ec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 00:32:36 +0000 Subject: [PATCH 045/110] Add filtered publishing --- publish/do.groovy | 66 ++++++++++++++++++++++++++++++++++++++---- publish/publish.groovy | 2 +- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 7844289..7ec8be8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -5,6 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" tempfolder = "${WORKSPACE}/temp" def resetfolders() { @@ -14,6 +15,9 @@ def resetfolders() { dir(releaserepofolder) { deleteDir() } + dir(filteredreleaserepofolder) { + deleteDir() + } dir(tempfolder) { deleteDir() } @@ -59,7 +63,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } } -def publish(String customer) { +def publish(String customer, String finalreleaserepofolder = releaserepofolder) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' @@ -67,15 +71,41 @@ def publish(String customer) { def String FTP_USER='ftp@globalinkjetsystems.com' def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' - dir(releaserepofolder) { + dir(finalreleaserepofolder) { summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash - ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ summaryappend(", Transferred ") } } +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdir(); + } + dirFrom.eachFile(FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def filteredpublish(itempathstopublish, customer) { + for (String itempath : itempathstopublish) { + def srcpath = "${releaserepofolder}/${itempath}" + File srcpathfile = new File(srcpath) + def dstpath = "${filteredreleaserepofolder}/${itempath}" + File dstpathfile = new File(dstpath) + def dstpathdirname = dstpathfile.getParent() + copydirectory(srcpath, dstpath) + } + publish(customer, filteredreleaserepofolder) +} + def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" @@ -102,11 +132,25 @@ def stripgithistory() { } } -def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + } + return paths +} + +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() preconditions() - + // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) @@ -122,9 +166,19 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin recallcomponent(releaseinfo, softwarerelease) } stripgithistory() - publish(customer) + + itempathstopublish = convertitemstopublish(itemstopublishstring) + if (itempathstopublish.isEmpty()) { + publish(customer) + } else { + filteredpublish(itempathstopublish, customer) + } return [summarystring, markunstable, unstablereason] } +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(releasemeta, "", customer, softwarerelease, maxdepth) +} + return this; diff --git a/publish/publish.groovy b/publish/publish.groovy index a703c9e..b478a5d 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -44,7 +44,7 @@ pipeline { steps { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' From 9fad59a8a8c08b0fac03e5160d21212e0cdb9874 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 07:44:00 +0000 Subject: [PATCH 046/110] Add filtered publishing --- publish/do.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/do.groovy b/publish/do.groovy index 7ec8be8..996123c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -101,6 +101,7 @@ def filteredpublish(itempathstopublish, customer) { def dstpath = "${filteredreleaserepofolder}/${itempath}" File dstpathfile = new File(dstpath) def dstpathdirname = dstpathfile.getParent() + echo "Copying recursively from ${srcpath} to ${dstpath}" copydirectory(srcpath, dstpath) } publish(customer, filteredreleaserepofolder) @@ -140,6 +141,7 @@ def convertitemstopublish(String itemstopublishstring) { def String[] pathitems = item.split('#') def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" paths.add(path) + echo "Selected item to publish: ${path}" } return paths } From 9ee093481dc01085da4318e4ce3e12a32e28270a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:34:07 +0000 Subject: [PATCH 047/110] Add filtered publishing --- publish/do.groovy | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 996123c..572c2e3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -41,6 +41,16 @@ def summaryappend(String summary) { summarystring += summary } +def guessreporoot(componentURL) { + if (env.REPOROOT || env.REPOROOT.length() == 0) { + return '' + } + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + def reporoot = "${REPOROOT}/${guess}" + echo "Guessing reporoot is: ${reporoot}" + return reporoot +} + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder @@ -56,7 +66,8 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { publishablePaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone + def reporoot = guessreporoot(componentURL) + gh.gitCheckout(componentURL, revision, repofolder, reporoot, 1, true, publishablePaths) // Shallow clone } else { summaryappend("None ") From 6a334f56f12470738342f88bdb58c28d06c317db Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:42:18 +0000 Subject: [PATCH 048/110] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 572c2e3..ade1c3f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -42,7 +42,7 @@ def summaryappend(String summary) { } def guessreporoot(componentURL) { - if (env.REPOROOT || env.REPOROOT.length() == 0) { + if (!env.REPOROOT || env.REPOROOT.size() == 0) { return '' } def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') From e328e823e4db25723952a5a900c20c9bd7a9cbcd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:47:47 +0000 Subject: [PATCH 049/110] Add filtered publishing --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index ade1c3f..14286b9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -42,11 +42,11 @@ def summaryappend(String summary) { } def guessreporoot(componentURL) { - if (!env.REPOROOT || env.REPOROOT.size() == 0) { + if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { return '' } def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPOROOT}/${guess}" + def reporoot = "${REPO_ROOT}/${guess}" echo "Guessing reporoot is: ${reporoot}" return reporoot } From 94a966c7cd4e055acf894b34a4edd4bcdd3d4740 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:52:40 +0000 Subject: [PATCH 050/110] Add filtered publishing --- publish/do.groovy | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 14286b9..e637e8f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -41,14 +41,9 @@ def summaryappend(String summary) { summarystring += summary } -def guessreporoot(componentURL) { - if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { - return '' - } +def guessreporootsuffix(componentURL) { def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPO_ROOT}/${guess}" - echo "Guessing reporoot is: ${reporoot}" - return reporoot + return guess } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") @@ -66,8 +61,8 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { publishablePaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } - def reporoot = guessreporoot(componentURL) - gh.gitCheckout(componentURL, revision, repofolder, reporoot, 1, true, publishablePaths) // Shallow clone + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckout(componentURL, revision, reporootsuffix, reporoot, 1, true, publishablePaths) // Shallow clone } else { summaryappend("None ") From 944f732e896a578636bc72be9851eba8032aa0ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:54:45 +0000 Subject: [PATCH 051/110] Add filtered publishing --- versioncontrol/git.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f7d10bb..9a313f2 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,6 +19,16 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } +def guessreporoot(componentURL) { + if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { + return '' + } + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + def reporoot = "${REPO_ROOT}/${guess}" + echo "Guessing reporoot is: ${reporoot}" + return reporoot +} + def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRootSuffix != "") { From 7c4d1df78d2022f6ba5fcf626d45257d35fee93c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:56:32 +0000 Subject: [PATCH 052/110] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index e637e8f..20e332a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -62,7 +62,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { summaryappend("${publishableitem.filepath},") } def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckout(componentURL, revision, reporootsuffix, reporoot, 1, true, publishablePaths) // Shallow clone + gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablePaths) // Shallow clone } else { summaryappend("None ") From df095fec9622a63f8591a9a5d65d11979c6af685 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:04:52 +0000 Subject: [PATCH 053/110] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 20e332a..c3d229e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -108,7 +108,7 @@ def filteredpublish(itempathstopublish, customer) { File dstpathfile = new File(dstpath) def dstpathdirname = dstpathfile.getParent() echo "Copying recursively from ${srcpath} to ${dstpath}" - copydirectory(srcpath, dstpath) + copydirectory(srcpathfile, dstpathfile) } publish(customer, filteredreleaserepofolder) } From 889d2b87977862468d0358e752395773537e10e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:20:02 +0000 Subject: [PATCH 054/110] Add filtered publishing --- publish/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c3d229e..eea1015 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -88,7 +88,7 @@ def publish(String customer, String finalreleaserepofolder = releaserepofolder) def copydirectory(File dirFrom, File dirTo) { if (!dirTo.exists()) { - dirTo.mkdir(); + dirTo.mkdirs(); } dirFrom.eachFile(FILES) { File source -> File target = new File(dirTo, source.getName()) @@ -103,12 +103,12 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { def srcpath = "${releaserepofolder}/${itempath}" - File srcpathfile = new File(srcpath) + def File srcpathfile = new File(srcpath) def dstpath = "${filteredreleaserepofolder}/${itempath}" - File dstpathfile = new File(dstpath) - def dstpathdirname = dstpathfile.getParent() - echo "Copying recursively from ${srcpath} to ${dstpath}" - copydirectory(srcpathfile, dstpathfile) + def File dstpathfile = new File(dstpath) + def dstpathparentdir = dstpathfile.getParentFile() + echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getName()}" + copydirectory(srcpathfile, dstpathparentdir) } publish(customer, filteredreleaserepofolder) } From e4e83a02f747485eb5f2aa47e63aaa0c9757aea0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:26:40 +0000 Subject: [PATCH 055/110] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index eea1015..255fae9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -107,7 +107,7 @@ def filteredpublish(itempathstopublish, customer) { def dstpath = "${filteredreleaserepofolder}/${itempath}" def File dstpathfile = new File(dstpath) def dstpathparentdir = dstpathfile.getParentFile() - echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getName()}" + echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" copydirectory(srcpathfile, dstpathparentdir) } publish(customer, filteredreleaserepofolder) From 9160ff3e7c3600be0bda79ef55d94520151f5e11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:45:43 +0000 Subject: [PATCH 056/110] Add filtered publishing --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 255fae9..e5e7965 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,11 +90,11 @@ def copydirectory(File dirFrom, File dirTo) { if (!dirTo.exists()) { dirTo.mkdirs(); } - dirFrom.eachFile(FILES) { File source -> + dirFrom.eachFile(FileType.FILES) { File source -> File target = new File(dirTo, source.getName()) target.bytes = source.bytes } - dirFrom.eachFile(DIRECTORIES) { File source -> + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> File target = new File(dirTo, source.getName()) copyLargeDir(source, target) } From d868c21c174d8157ef3820dd0bd0ea1dc5699325 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:48:13 +0000 Subject: [PATCH 057/110] Add filtered publishing --- publish/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index e5e7965..7ce778c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -143,12 +143,14 @@ def convertitemstopublish(String itemstopublishstring) { def List paths = [] def String[] items = itemstopublishstring.split(',') + def String allpaths for(String item : items) { def String[] pathitems = item.split('#') def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" paths.add(path) - echo "Selected item to publish: ${path}" + allpaths += path } + echo "Selected item to publish: ${allpaths}" return paths } From 76f24173a3091d3c5b652159adb3769bfe917374 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:54:08 +0000 Subject: [PATCH 058/110] Add filtered publishing --- publish/do.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 7ce778c..c4bf7eb 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,3 +1,5 @@ +import groovy.io.FileType + markunstable = false unstablereason = "" @@ -143,14 +145,14 @@ def convertitemstopublish(String itemstopublishstring) { def List paths = [] def String[] items = itemstopublishstring.split(',') - def String allpaths + def String allpaths = "" for(String item : items) { def String[] pathitems = item.split('#') def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" paths.add(path) - allpaths += path + allpaths += "${path}\n" } - echo "Selected item to publish: ${allpaths}" + echo "Selected item to publish:\n${allpaths}" return paths } From 7f8b0cf8667f2df714f1aeefc187e0366b148d4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 12:54:50 +0000 Subject: [PATCH 059/110] Add filtered publishing --- publish/do.groovy | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c4bf7eb..749497a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,13 +104,16 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { - def srcpath = "${releaserepofolder}/${itempath}" - def File srcpathfile = new File(srcpath) - def dstpath = "${filteredreleaserepofolder}/${itempath}" - def File dstpathfile = new File(dstpath) - def dstpathparentdir = dstpathfile.getParentFile() - echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" - copydirectory(srcpathfile, dstpathparentdir) + def srcpathglob = "${releaserepofolder}/${itempath}" + def srcpaths = new FileNameFinder().getFileNames(srcpathglob) + for (String srcpath: srcpaths) { + def File srcpathfile = new File(srcpath) + def dstpath = "${filteredreleaserepofolder}/${itempath}" + def File dstpathfile = new File(dstpath) + def dstpathparentdir = dstpathfile.getParentFile() + echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" + copydirectory(srcpathfile, dstpathparentdir) + } } publish(customer, filteredreleaserepofolder) } From b822e39e2a591c86fe35ad4ed014fff8a7d5205a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:04:43 +0000 Subject: [PATCH 060/110] Add filtered publishing --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 749497a..7262a3a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,8 +104,8 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { - def srcpathglob = "${releaserepofolder}/${itempath}" - def srcpaths = new FileNameFinder().getFileNames(srcpathglob) + // Convert glob wildcard paths to real paths + def srcpaths = new FileNameFinder().getFileNames(releaserepofolder, itempath) for (String srcpath: srcpaths) { def File srcpathfile = new File(srcpath) def dstpath = "${filteredreleaserepofolder}/${itempath}" From f9a078c895c5facd1c3f8273d416907444169abe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:15:54 +0000 Subject: [PATCH 061/110] Add filtered publishing --- publish/do.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/do.groovy b/publish/do.groovy index 7262a3a..2d5ee05 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -105,7 +105,9 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { // Convert glob wildcard paths to real paths + echo "Finding files at base folder: ${releaserepofolder} and glob filter: ${itempath}" def srcpaths = new FileNameFinder().getFileNames(releaserepofolder, itempath) + echo "Paths found: ${srcpaths}" for (String srcpath: srcpaths) { def File srcpathfile = new File(srcpath) def dstpath = "${filteredreleaserepofolder}/${itempath}" From 36ca33077f74c74c11dc747d33e345a165a904ee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:34:40 +0000 Subject: [PATCH 062/110] Add filtered publishing --- publish/do.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 2d5ee05..732373e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,9 +104,14 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { - // Convert glob wildcard paths to real paths - echo "Finding files at base folder: ${releaserepofolder} and glob filter: ${itempath}" - def srcpaths = new FileNameFinder().getFileNames(releaserepofolder, itempath) + def defaultsrcpath = "${releaserepofolder}/${itempath}" + def srcpaths = [ defaultsrcpath ] + if (itempath.endsWith('**') { // Convert glob wildcard paths to real paths + File srcfilepath = new File(defaultsrcpath).getParentFile() + def globpattern = '**' + echo "Finding files at base folder: ${srcfilepath} and glob filter: ${globpattern}" + srcpaths = new FileNameFinder().getFileNames(releaserepofolder, globpattern) + } echo "Paths found: ${srcpaths}" for (String srcpath: srcpaths) { def File srcpathfile = new File(srcpath) From 36ea650ea7dfa7601b75ae3244bfb0a80f3390f7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:42:43 +0000 Subject: [PATCH 063/110] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 732373e..dcc88ca 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -106,7 +106,7 @@ def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { def defaultsrcpath = "${releaserepofolder}/${itempath}" def srcpaths = [ defaultsrcpath ] - if (itempath.endsWith('**') { // Convert glob wildcard paths to real paths + if (itempath.endsWith('**')) { // Convert glob wildcard paths to real paths File srcfilepath = new File(defaultsrcpath).getParentFile() def globpattern = '**' echo "Finding files at base folder: ${srcfilepath} and glob filter: ${globpattern}" From aa100d8731cea24e75714c07a85e86c011c4040b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:21:51 +0000 Subject: [PATCH 064/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 68 ++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index dcc88ca..a10fe51 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -25,6 +25,14 @@ def resetfolders() { } } +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + def preconditions() { markunstable = false unstablereason = "" @@ -49,7 +57,7 @@ def guessreporootsuffix(componentURL) { } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -57,14 +65,18 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo - def ArrayList publishablePaths = new ArrayList() + def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + if (itempathstopublish.size() == 0 || publishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } } + } + if (publishablepaths.size() > 0) { def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablePaths) // Shallow clone + gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone } else { summaryappend("None ") @@ -102,38 +114,22 @@ def copydirectory(File dirFrom, File dirTo) { } } -def filteredpublish(itempathstopublish, customer) { - for (String itempath : itempathstopublish) { - def defaultsrcpath = "${releaserepofolder}/${itempath}" - def srcpaths = [ defaultsrcpath ] - if (itempath.endsWith('**')) { // Convert glob wildcard paths to real paths - File srcfilepath = new File(defaultsrcpath).getParentFile() - def globpattern = '**' - echo "Finding files at base folder: ${srcfilepath} and glob filter: ${globpattern}" - srcpaths = new FileNameFinder().getFileNames(releaserepofolder, globpattern) - } - echo "Paths found: ${srcpaths}" - for (String srcpath: srcpaths) { - def File srcpathfile = new File(srcpath) - def dstpath = "${filteredreleaserepofolder}/${itempath}" - def File dstpathfile = new File(dstpath) - def dstpathparentdir = dstpathfile.getParentFile() - echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" - copydirectory(srcpathfile, dstpathparentdir) - } - } - publish(customer, filteredreleaserepofolder) +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" } -def recallcomponent(releaseinfo, String softwarerelease) { +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + +def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) + def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) summaryappend("\n") } @@ -183,19 +179,13 @@ def publishhierarchy(releasemeta, String itemstopublishstring, String customer, } echo versionuniquecomponent + def itempathstopublish = convertitemstopublish(itemstopublishstring) for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] - recallcomponent(releaseinfo, softwarerelease) + recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } stripgithistory() - itempathstopublish = convertitemstopublish(itemstopublishstring) - if (itempathstopublish.isEmpty()) { - publish(customer) - } else { - filteredpublish(itempathstopublish, customer) - } - return [summarystring, markunstable, unstablereason] } From e05f6a2da23d5c6d621365322c3fb3200edf3feb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:26:14 +0000 Subject: [PATCH 065/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 8 -------- 1 file changed, 8 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index a10fe51..62dea7c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -114,14 +114,6 @@ def copydirectory(File dirFrom, File dirTo) { } } -def uniquereleasefolder(String softwarerelease, releasemeta) { - return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" -} - -def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { - return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" -} - def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) From 755172bc503eb7e7bd9b89c7c67c282b9815e1c2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:32:52 +0000 Subject: [PATCH 066/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/publish/do.groovy b/publish/do.groovy index 62dea7c..53fc791 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -65,6 +65,10 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo + + echo "itempathstopublish: ${itempathstopublish}" + echo "publishableitems: ${publishableitems}" + def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { From 14a6291d3dd39cbddfad5c92cc2715a7c105c4cc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:40:16 +0000 Subject: [PATCH 067/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 53fc791..f4aad08 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -66,13 +66,10 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo - echo "itempathstopublish: ${itempathstopublish}" - echo "publishableitems: ${publishableitems}" - def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - if (itempathstopublish.size() == 0 || publishableitem in itempathstopublish) { + if (itempathstopublish.size() == 0 || publishableitem.filepath in itempathstopublish) { publishablepaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } From 19932d9f1549f8d9cfc449f07e9d4127d94b8492 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:50:46 +0000 Subject: [PATCH 068/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/do.groovy b/publish/do.groovy index f4aad08..9073b85 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -69,6 +69,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { + echo "publishableitem: ${publishableitem.filepath}" if (itempathstopublish.size() == 0 || publishableitem.filepath in itempathstopublish) { publishablepaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") From d112e4f7d187ee601f6ecd1d9c1ba1b12808dafc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:59:06 +0000 Subject: [PATCH 069/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 9073b85..80113f3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -25,8 +25,12 @@ def resetfolders() { } } +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + def uniquereleasefolder(String softwarerelease, releasemeta) { - return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + return "Software/${PRODUCT_NAME} ${softwarerelease}/${productuniquefolder(releasemeta)}" } def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { @@ -69,8 +73,9 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - echo "publishableitem: ${publishableitem.filepath}" - if (itempathstopublish.size() == 0 || publishableitem.filepath in itempathstopublish) { + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { publishablepaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } From 5cac7a574bdba3dcf6dd31516ff249d7813eebb1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 15:53:27 +0000 Subject: [PATCH 070/110] Add filtered publishing - different approach for filtering --- publish/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/do.groovy b/publish/do.groovy index 80113f3..1a54b23 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -184,6 +184,7 @@ def publishhierarchy(releasemeta, String itemstopublishstring, String customer, recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } stripgithistory() + publish(customer) return [summarystring, markunstable, unstablereason] } From fa989ef12f6ac06ac840cc1b562e9590fd242883 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 22 Mar 2019 13:36:45 +0000 Subject: [PATCH 071/110] SMB-119 - Not barfing when no substitions occurred --- versionstamping/versionstamper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 3e59c6f..4c380a6 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -26,15 +26,18 @@ def main(argv): print("Could not find file: {0}\n".format(filetoconvert)) return 1 + count = 0 with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: fileread = c.read() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) - #print ("DEBUG: %s "%converted) - print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) f.write(converted) + if count == 0: + return 1 return 0 if __name__ == "__main__": From d1741859bc1a0c42885e702edd9b9fce8dd7d31a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:37:17 +0000 Subject: [PATCH 072/110] Add some checking job scripts --- check/build_meta.groovy | 86 +++++++++++++++++++++++++++++++++++++++++ check/do_meta.groovy | 64 ++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 check/build_meta.groovy create mode 100644 check/do_meta.groovy diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..34eed98 --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,86 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..75353da --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,64 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + def offsetpath = buildinfo.offsetpath + versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; From d75c59e92ada16a48032b23a85d5d5dc89ea097d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:44:08 +0000 Subject: [PATCH 073/110] Add some checking job scripts --- check/do_meta.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 75353da..6712a11 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,14 +47,15 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() - // Walk the meta information to a max depth, looking for components to release + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath - versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 86fc6f3b5eb22b2dd35a792fce3acfc485a045ec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:07:24 +0000 Subject: [PATCH 074/110] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 6712a11..e01aa41 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 64fecfb6b3208ae982a277d34d0bcfe3dae893a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:12:31 +0000 Subject: [PATCH 075/110] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index e01aa41..7fce1eb 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 741a9f966c0822e4b678165cab07c6f1dbfb3fe0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:22:58 +0000 Subject: [PATCH 076/110] Add some checking job scripts --- check/do_meta.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 7fce1eb..cb19ddf 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,6 +47,18 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = buildinfo.meta.componentversionuniqueidentifier() + def offsetpath = buildinfo.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" From 401ce43ea3d42a469fa41d588ab5e3a464059b0e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:25:25 +0000 Subject: [PATCH 077/110] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index cb19ddf..8935ca5 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -64,8 +64,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - def offsetpath = buildinfo.offsetpath + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.offsetpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } From c59af13d9c6c6668d7486eb391c1daa4b8dc81ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:29:23 +0000 Subject: [PATCH 078/110] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 8935ca5..7bfe2d1 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = buildinfo.meta.componentversionuniqueidentifier() - def offsetpath = buildinfo.offsetpath + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } From 6555898bb55b91d61dae523daa9af06c6d6e3411 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:51:03 +0000 Subject: [PATCH 079/110] Add some checking job scripts --- meta/folder.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d751f5e..a18e8d7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,6 +687,18 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } + Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + } + } else { + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetacollection + } + Map releasewalkerunique(int maxdepth, String startpath = "") { def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) @@ -694,7 +706,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) } return releasemetauniquecollection @@ -707,7 +719,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetaversionuniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) } return releasemetaversionuniquecollection From 1ed3b8214c7e945d3ebe452561e2131cc0d99e34 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:54:50 +0000 Subject: [PATCH 080/110] Add some checking job scripts --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a18e8d7..a8ffa41 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,14 +687,14 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } - Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } } else { - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } return releasemetacollection } From 323da9fdf3edf65330275c026cc8828858b56fdb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:57:14 +0000 Subject: [PATCH 081/110] Add some checking job scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8ffa41..1bf855c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -690,7 +690,7 @@ class MetaInfo extends GISJsonSerialised { Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { releasemetacollection.put(uniquestring, relatedmetainfo) } } else { From 9e0f24f0d26d9c2f511d9cdbc32f6bfb6d4fd0f3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 12:21:15 +0000 Subject: [PATCH 082/110] Create a script to copy logic to a build job - so that it will be stamped during release --- copy/logic.groovy | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 copy/logic.groovy diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..6f8f633 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,56 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/logic", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying logic artifacts') { + steps { + script { + env.LOGICDEST = "${WORKSPACE}/logic" + if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["logic/**"]) + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} From 88e955e7653d8b2dc4a42a333b28a385911f191d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Mar 2019 14:21:19 +0000 Subject: [PATCH 083/110] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6f8f633..d951ffa 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -35,6 +35,9 @@ pipeline { script { def artifactlist = meta.generateartifacts(["logic/**"]) archiveArtifacts "${artifactlist}" + sh """ + cp -v -R logic/meta/info.json meta/info.json + """ } } } From 3c0b2178a810d4648220e5b258719a8d8a05ec3d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Mar 2019 14:25:25 +0000 Subject: [PATCH 084/110] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index d951ffa..845f676 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -34,10 +34,10 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["logic/**"]) - archiveArtifacts "${artifactlist}" sh """ cp -v -R logic/meta/info.json meta/info.json """ + archiveArtifacts "${artifactlist}" } } } From 6a600862582d3b19a59e358007c038d471c3bc05 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 08:56:29 +0000 Subject: [PATCH 085/110] Handle the artifact source parameter --- import/importer.groovy | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/import/importer.groovy b/import/importer.groovy index 5c996b4..40151e1 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -35,4 +35,18 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + switch (source) { + case "Jenkins job artifact": { + return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + case "Release repository": { + return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + return this; From 11b7fe881da3416d70bde83001592cc56f7aa9d8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 08:59:37 +0000 Subject: [PATCH 086/110] Handle the artifact source parameter --- import/importer.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 40151e1..abf3b2f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -37,12 +37,10 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { - case "Jenkins job artifact": { + case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) - } - case "Release repository": { + case "Release repository": return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) - } default: error("Unknown type of artifact source: ${source}") } From 05afef18481f42696eded8a57404336f63cfd98e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 09:16:18 +0000 Subject: [PATCH 087/110] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 845f676..fae69b4 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -13,6 +13,7 @@ pipeline { script { cleanfolders = [ "${WORKSPACE}/logic", + "${WORKSPACE}/embedded_logic", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -33,9 +34,10 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["logic/**"]) + def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - cp -v -R logic/meta/info.json meta/info.json + rm -v meta/info.json + mv -v logic/embedded_logic . """ archiveArtifacts "${artifactlist}" } From ff733458f94621a24c3af027bd2ae73db169561d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 09:19:44 +0000 Subject: [PATCH 088/110] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index fae69b4..6e97510 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,8 +36,8 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v meta/info.json - mv -v logic/embedded_logic . + rm -v -rf meta + mv -v logic/* . """ archiveArtifacts "${artifactlist}" } From ec33ceb812e97971316553e88676b32304aa8162 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 09:06:33 +0000 Subject: [PATCH 089/110] Import artifacts from repo --- import/importer.groovy | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index abf3b2f..e605743 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,4 @@ -versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( @@ -35,6 +35,18 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + gh.gitCheckout(rooturl, rootrevision, destpath) + + def importedmeta = meta.importrelated(destpath) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": From 51ba9bc94e85d07a464621c1d7d1c8d15f5ec673 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 09:25:47 +0000 Subject: [PATCH 090/110] Import artifacts from repo --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index e605743..fc8395f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,5 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( From 3745e27b9cb5d199044a693bf01f25e686257b7f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 13:57:21 +0000 Subject: [PATCH 091/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 9 +++++++++ meta/folder.groovy | 10 +++++++++- release/release.groovy | 3 +-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index fc8395f..133edba 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,6 +42,15 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.gitCheckout(rooturl, rootrevision, destpath) def importedmeta = meta.importrelated(destpath) + + for(associatedmeta in getassociatedmeta) { + def cloneurl = associatedmeta.releaseinfo.releaserepoURL + def clonerevision = associatedmeta.releasetag() + def clonepath = "${destpath}/${associatedmeta.offsetpath}" + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckout(cloneurl, rootrevision, clonepath) + } + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } diff --git a/meta/folder.groovy b/meta/folder.groovy index 1bf855c..053c38e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -477,6 +477,10 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } + ArrayList getassociatedmeta() { + return associatedmeta + } + List configurationinfoserialised() { def configuration = [] for (configurationitem in configurationinfo) { @@ -684,7 +688,11 @@ class MetaInfo extends GISJsonSerialised { } String componentversionuniqueidentifier() { - return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp + return componentidentifier() + "_" + buildstampinfo.buildstamp + } + + String releasetag() { + return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber } Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { diff --git a/release/release.groovy b/release/release.groovy index b0530be..3b5321d 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,8 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" - OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" + OTHER_RELEASE_TAGS = "${buildmeta.releasetag()},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason From 6056ba5874063695b74e72cee121b961e1cb3173 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 14:03:22 +0000 Subject: [PATCH 092/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 133edba..09451ec 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,7 +43,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - for(associatedmeta in getassociatedmeta) { + for(associatedmeta in importedmeta.getassociatedmeta()) { def cloneurl = associatedmeta.releaseinfo.releaserepoURL def clonerevision = associatedmeta.releasetag() def clonepath = "${destpath}/${associatedmeta.offsetpath}" From 659859878d3aad5e532b7c2136d0c2b328030d40 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 14:19:39 +0000 Subject: [PATCH 093/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 09451ec..2392b83 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,10 +43,12 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - for(associatedmeta in importedmeta.getassociatedmeta()) { - def cloneurl = associatedmeta.releaseinfo.releaserepoURL - def clonerevision = associatedmeta.releasetag() - def clonepath = "${destpath}/${associatedmeta.offsetpath}" + for(relatedmetainfo in importedmeta.getassociatedmeta()) { + def relatedmeta = relatedmetainfo.meta + def relatedoffsetpath = relatedmetainfo.offsetpath + def cloneurl = relatedmeta.releaseinfo.releaserepoURL + def clonerevision = "refs/tags/${relatedmeta.releasetag()}" + def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" gh.gitCheckout(cloneurl, rootrevision, clonepath) } From 9160063d947034622003e95e24eb2e31090e413e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 14:31:49 +0000 Subject: [PATCH 094/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 2392b83..c554173 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,7 +43,8 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - for(relatedmetainfo in importedmeta.getassociatedmeta()) { + relatedimportinfo = importedmeta.getassociatedmeta() + for(relatedmetainfo in relatedimportinfo) { def relatedmeta = relatedmetainfo.meta def relatedoffsetpath = relatedmetainfo.offsetpath def cloneurl = relatedmeta.releaseinfo.releaserepoURL From ab18f25218f5afe0402fbc3e9f017ebdbf4af0a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 16:45:17 +0000 Subject: [PATCH 095/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 3 ++- meta/folder.groovy | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index c554173..cdc2e1a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,7 +43,8 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - relatedimportinfo = importedmeta.getassociatedmeta() + relatedimportinfo = importedmeta.getassociated() + echo "Cloning associated repos" for(relatedmetainfo in relatedimportinfo) { def relatedmeta = relatedmetainfo.meta def relatedoffsetpath = relatedmetainfo.offsetpath diff --git a/meta/folder.groovy b/meta/folder.groovy index 053c38e..987041b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -477,7 +477,7 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } - ArrayList getassociatedmeta() { + List getassociated() { return associatedmeta } @@ -635,12 +635,11 @@ class MetaInfo extends GISJsonSerialised { ] return data } - static String folder() { return METAFOLDERNAME } - + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" From 5006d6cd184fc11e30f1c99edf4dc390cb0e6d9a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 16:48:58 +0000 Subject: [PATCH 096/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index cdc2e1a..78f89cc 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -39,6 +39,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" gh.gitCheckout(rooturl, rootrevision, destpath) def importedmeta = meta.importrelated(destpath) From e23d564cb589081f986f9181d48e0a3e0b9fd701 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 16:57:54 +0000 Subject: [PATCH 097/110] When importing artifacts from repo, clone subcomponents --- meta/folder.groovy | 6 +++++- release/release.groovy | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 987041b..e805118 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -690,10 +690,14 @@ class MetaInfo extends GISJsonSerialised { return componentidentifier() + "_" + buildstampinfo.buildstamp } - String releasetag() { + String globalreleasetag() { return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber } + String releasetag() { + return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber + } + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] diff --git a/release/release.groovy b/release/release.groovy index 3b5321d..79c0234 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,7 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - OTHER_RELEASE_TAGS = "${buildmeta.releasetag()},${OTHER_RELEASE_TAGS}" + OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason From 7dd532095cc0e8d597fb4cf41c9b1464b0708ca1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 17:00:25 +0000 Subject: [PATCH 098/110] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 78f89cc..c6b80a4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -53,7 +53,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckout(cloneurl, rootrevision, clonepath) + gh.gitCheckout(cloneurl, clonerevision, clonepath) } if (incrementversionnumber) { From 8cd99c0927f3a215c996e88c8896cb5a5c12077e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:33:55 +0000 Subject: [PATCH 099/110] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- import/importer.groovy | 16 +++++++++++++++- meta/folder.groovy | 10 +++++++++- publish/do.groovy | 15 +-------------- versioncontrol/git.groovy | 13 +++++++++++++ 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index c6b80a4..c945ec0 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -36,6 +36,12 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def rewritemetafileasreleased(meta, String filepath) { + // Modify the meta info to say that is has already been released + meta.releaseinfo.markreleased() + meta.saveJsonSerialised(filepath) +} + def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() @@ -54,8 +60,16 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" gh.gitCheckout(cloneurl, clonerevision, clonepath) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) } - + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${relatedmeta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } diff --git a/meta/folder.groovy b/meta/folder.groovy index e805118..ec011ae 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -219,11 +219,13 @@ class BuildStampInfo extends GISJsonSerialised { class ReleaseInfo extends GISJsonSerialised { String releaserepoURL ArrayList releaseitems + boolean released - ReleaseInfo(script, releaserepoURL = "") { + ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { super(script) this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() + this.released = released } void set(String releaserepoURL = "") { @@ -234,6 +236,10 @@ class ReleaseInfo extends GISJsonSerialised { releaseitems.add(releaseitem) } + void markreleased() { + this.released = true + } + List releaseitemssserialised() { def items = [] for (ri in releaseitems) { @@ -248,12 +254,14 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } + released ?= rawmeta['released'] : false } LinkedHashMap serialisationdata() { def data = [ 'releaserepoURL' : releaserepoURL, 'releaseitems' : releaseitemssserialised(), + 'released' : released ] return data } diff --git a/publish/do.groovy b/publish/do.groovy index 1a54b23..b63b15c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -133,19 +133,6 @@ def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { summaryappend("\n") } -def stripgithistory() { - echo "Stripping git history out of ${releaserepofolder}" - dir(releaserepofolder) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rvf XX - find . -name ".gitignore" | xargs -iXX rm -rvf XX - find . -name ".gitmodules" | xargs -iXX rm -rvf XX - find . -iname ".gitattributes" | xargs -iXX rm -rvf XX - """ - } -} - def convertitemstopublish(String itemstopublishstring) { def List paths = [] @@ -183,7 +170,7 @@ def publishhierarchy(releasemeta, String itemstopublishstring, String customer, def releaseinfo = componentReleaseInfoVersionUnique[key] recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } - stripgithistory() + gh.stripgithistory(releaserepofolder) publish(customer) return [summarystring, markunstable, unstablereason] diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 9a313f2..7a1dd17 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -89,4 +89,17 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { return true } +def stripgithistory(String rootfolder) { + echo "Stripping git history out of ${rootfolder}" + dir(rootfolder) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX + """ + } +} + return this From 5659c8eeb41dbc2dcd48b1a7659ac1581e937287 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:36:35 +0000 Subject: [PATCH 100/110] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ec011ae..be85584 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -254,7 +254,7 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } - released ?= rawmeta['released'] : false + released = rawmeta['released'] ?= false } LinkedHashMap serialisationdata() { From a5dc59e6b297e8728cb9bcb5723f65f9427318c1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:38:02 +0000 Subject: [PATCH 101/110] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index be85584..a868855 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -254,7 +254,7 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } - released = rawmeta['released'] ?= false + released = rawmeta['released'] ?: false } LinkedHashMap serialisationdata() { From 49783e0f468deb3318ce65a0024df3a32c200984 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:43:58 +0000 Subject: [PATCH 102/110] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index c945ec0..a23ee83 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -66,7 +66,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, rewritemetafileasreleased(relatedmeta, metafilepath) } gh.stripgithistory(destpath) - def metafilepath = "${destpath}/${relatedmeta.metafilepath()}" + def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(meta, metafilepath) From 86613fcb2cf9cab5dc11ec847a65dcfef9f67195 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 19:43:15 +0000 Subject: [PATCH 103/110] When releasing, take into account already-released components --- release/do.groovy | 61 ++++++++++++++++++++++++--------------- versioncontrol/git.groovy | 8 ++--- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 604581a..fd2b06b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -172,22 +172,29 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele return taglist } -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false +def String releasegitgetshacode(String tag) { + def existingshacode = "" // Check if tag exists already def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() + existingshacode = readFile(file: gitrevparseoutputfile).trim() + } + return existingshacode +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def String existingshacode = releasegitgetshacode(tag) + if (existingshacode) { // Tag exists if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") summaryappend("(${tag} - Could not add!)") - } - else { + } else { summaryappend("(${tag} - Already added)") } - } - else { // If tag does not exist already + } else { // If tag does not exist already // Tag sh("git tag \"${tag}\"") summaryappend("(${tag})") @@ -270,26 +277,35 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp - def String buildstamptag = "${buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" + def String buildstamptag = buildmeta.releasetag() dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - releasegitadd() + def boolean alreadyreleased = buildmeta.releaseinfo.released + echo "Already released: ${alreadyreleased}" - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamptag) - } - else { - releasegitreset() - summaryappend("No-commit,") - } + def String postcommitshacode + def boolean changed = false + if (alreadyreleased) { + summaryappend("Already-released,") + postcommitshacode = releasegitgetshacode(buildstamptag) + echo "Previous release sha code: ${postcommitshacode}" + } else { + releasegitadd() - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" + changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamptag) + } else { + releasegitreset() + summaryappend("No-commit,") + } + + postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + } ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) @@ -299,7 +315,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamptag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamptag}" + def String srcbuildstamptag = buildmeta.globalreleasetag() def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { @@ -308,8 +324,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source if (tagadded) { sourcegitpush(dryrunpush) } - } - else { + } else { summaryappend(",No Src ") } } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 7a1dd17..ba4d96b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -94,10 +94,10 @@ def stripgithistory(String rootfolder) { dir(rootfolder) { sh """#!/bin/bash pwd - find . -type d -name ".git" | xargs -iXX rm -rvf XX - find . -name ".gitignore" | xargs -iXX rm -rvf XX - find . -name ".gitmodules" | xargs -iXX rm -rvf XX - find . -iname ".gitattributes" | xargs -iXX rm -rvf XX + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX """ } } From 6eb0f77114338dd00296ebfe7a8837c62dec34a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 19:58:37 +0000 Subject: [PATCH 104/110] Recursively mark all associated items of meta as released --- import/importer.groovy | 2 +- meta/folder.groovy | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index a23ee83..34707bc 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -38,7 +38,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def rewritemetafileasreleased(meta, String filepath) { // Modify the meta info to say that is has already been released - meta.releaseinfo.markreleased() + meta.markreleased() meta.saveJsonSerialised(filepath) } diff --git a/meta/folder.groovy b/meta/folder.groovy index a868855..346b2ee 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -489,6 +489,13 @@ class MetaInfo extends GISJsonSerialised { return associatedmeta } + void markreleased() { + releaseinfo.markreleased() + for (associated in associatedmeta) { + associated.meta.releaseinfo.markreleased() + } + } + List configurationinfoserialised() { def configuration = [] for (configurationitem in configurationinfo) { From a756df369aae75957a999570992da95b59335fd1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:44:12 +0000 Subject: [PATCH 105/110] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ba4d96b..a474b6e 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -29,15 +29,50 @@ def guessreporoot(componentURL) { return reporoot } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRootSuffix != "") { - reporootpath = "${reporootenv.trim()}/${repoRootSuffix}" - echo "Using release repo: ${reporootpath}" - } else { - reporootpath = "" +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" - } + } else { + def reporootname = uniquereporootname(gitURL) + def reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootname}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootname}" + } + return reporootsubfolder +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', userRemoteConfigs: [ From df087201a3f9bb54c2b65561f4c95fc21be1c344 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:48:35 +0000 Subject: [PATCH 106/110] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a474b6e..bd1b15e 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,16 +19,6 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def guessreporoot(componentURL) { - if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { - return '' - } - def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPO_ROOT}/${guess}" - echo "Guessing reporoot is: ${reporoot}" - return reporoot -} - def String uniquereporootname(String gitURL) { def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" @@ -68,6 +58,7 @@ def String checkouttoreporoot(String gitURL) { } else { echo "Found ${gitURL} checked out at ${reporootname}" } + } return reporootsubfolder } From 9dd09e4444eabe13100dec7a3dbca9b5825b5f20 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:49:54 +0000 Subject: [PATCH 107/110] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index bd1b15e..ea3501c 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,7 +34,7 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - def reporootsubfolder = "${reporootenv}/${reporootname}" + reporootsubfolder = "${reporootenv}/${reporootname}" if (!fileExists(reporootsubfolder)) { echo "Checking out ${gitURL} to ${reporootname}" From d1e8531b5ad42b0997fc53036a555e9c25c961fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:52:55 +0000 Subject: [PATCH 108/110] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ea3501c..432316f 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,10 +34,10 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootname}" + reporootsubfolder = "${reporootenv}/${reporootsubfolder}" if (!fileExists(reporootsubfolder)) { - echo "Checking out ${gitURL} to ${reporootname}" + echo "Checking out ${gitURL} to ${reporootsubfolder}" checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -56,7 +56,7 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootname}" + echo "Found ${gitURL} checked out at ${reporootsubfolder}" } } return reporootsubfolder From add368c009a6f99a15c7bf6d2de25af96be5600d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:56:01 +0000 Subject: [PATCH 109/110] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 432316f..5599dc6 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,7 +34,7 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootsubfolder}" + reporootsubfolder = "${reporootenv}/${reporootname}" if (!fileExists(reporootsubfolder)) { echo "Checking out ${gitURL} to ${reporootsubfolder}" From 34a79713cdf76c175386f954080896955f6981f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Apr 2019 09:01:42 +0100 Subject: [PATCH 110/110] Check contents of reporoot repo --- versioncontrol/git.groovy | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 5599dc6..253ffee 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -56,7 +56,20 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootsubfolder}" + echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status" + dir(reporootsubfolder) { + if (isUnix()) { + sh """ + git status + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + bat """ + git status + if errorlevel 1 EXIT /b 1 + """ + } + } } } return reporootsubfolder