From fa989ef12f6ac06ac840cc1b562e9590fd242883 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 22 Mar 2019 13:36:45 +0000 Subject: [PATCH 01/12] SMB-119 - Not barfing when no substitions occurred --- versionstamping/versionstamper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 3e59c6f..4c380a6 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -26,15 +26,18 @@ def main(argv): print("Could not find file: {0}\n".format(filetoconvert)) return 1 + count = 0 with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: fileread = c.read() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) - #print ("DEBUG: %s "%converted) - print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) f.write(converted) + if count == 0: + return 1 return 0 if __name__ == "__main__": From d1741859bc1a0c42885e702edd9b9fce8dd7d31a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:37:17 +0000 Subject: [PATCH 02/12] Add some checking job scripts --- check/build_meta.groovy | 86 +++++++++++++++++++++++++++++++++++++++++ check/do_meta.groovy | 64 ++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 check/build_meta.groovy create mode 100644 check/do_meta.groovy diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..34eed98 --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,86 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..75353da --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,64 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + def offsetpath = buildinfo.offsetpath + versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; From d75c59e92ada16a48032b23a85d5d5dc89ea097d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:44:08 +0000 Subject: [PATCH 03/12] Add some checking job scripts --- check/do_meta.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 75353da..6712a11 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,14 +47,15 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() - // Walk the meta information to a max depth, looking for components to release + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath - versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 86fc6f3b5eb22b2dd35a792fce3acfc485a045ec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:07:24 +0000 Subject: [PATCH 04/12] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 6712a11..e01aa41 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 64fecfb6b3208ae982a277d34d0bcfe3dae893a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:12:31 +0000 Subject: [PATCH 05/12] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index e01aa41..7fce1eb 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 741a9f966c0822e4b678165cab07c6f1dbfb3fe0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:22:58 +0000 Subject: [PATCH 06/12] Add some checking job scripts --- check/do_meta.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 7fce1eb..cb19ddf 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,6 +47,18 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = buildinfo.meta.componentversionuniqueidentifier() + def offsetpath = buildinfo.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" From 401ce43ea3d42a469fa41d588ab5e3a464059b0e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:25:25 +0000 Subject: [PATCH 07/12] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index cb19ddf..8935ca5 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -64,8 +64,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - def offsetpath = buildinfo.offsetpath + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.offsetpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } From c59af13d9c6c6668d7486eb391c1daa4b8dc81ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:29:23 +0000 Subject: [PATCH 08/12] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 8935ca5..7bfe2d1 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = buildinfo.meta.componentversionuniqueidentifier() - def offsetpath = buildinfo.offsetpath + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } From 6555898bb55b91d61dae523daa9af06c6d6e3411 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:51:03 +0000 Subject: [PATCH 09/12] Add some checking job scripts --- meta/folder.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d751f5e..a18e8d7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,6 +687,18 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } + Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + } + } else { + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetacollection + } + Map releasewalkerunique(int maxdepth, String startpath = "") { def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) @@ -694,7 +706,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) } return releasemetauniquecollection @@ -707,7 +719,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetaversionuniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) } return releasemetaversionuniquecollection From 1ed3b8214c7e945d3ebe452561e2131cc0d99e34 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:54:50 +0000 Subject: [PATCH 10/12] Add some checking job scripts --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a18e8d7..a8ffa41 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,14 +687,14 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } - Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } } else { - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } return releasemetacollection } From 323da9fdf3edf65330275c026cc8828858b56fdb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:57:14 +0000 Subject: [PATCH 11/12] Add some checking job scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8ffa41..1bf855c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -690,7 +690,7 @@ class MetaInfo extends GISJsonSerialised { Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { releasemetacollection.put(uniquestring, relatedmetainfo) } } else { From 9e0f24f0d26d9c2f511d9cdbc32f6bfb6d4fd0f3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 12:21:15 +0000 Subject: [PATCH 12/12] Create a script to copy logic to a build job - so that it will be stamped during release --- copy/logic.groovy | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 copy/logic.groovy diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..6f8f633 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,56 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/logic", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying logic artifacts') { + steps { + script { + env.LOGICDEST = "${WORKSPACE}/logic" + if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["logic/**"]) + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +}