From 9fda4bce8b159fcba75abc8cf6a91f3d7d18950e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Dec 2018 13:40:05 +0000 Subject: [PATCH 01/28] Turn off fingerprinting --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 0155c7f..093ddef 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -7,7 +7,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def artifactKey = installerDetails.get(installer)[0] (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy) artifactFetched.put(artifactKey, importmeta) } } From c41563f0c216b19902af393136ce2d4cba468f60 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:33:17 +0000 Subject: [PATCH 02/28] Basebuildnumber is now an external parameter --- versionnumbering/generate.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 7e5568a..f26806d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,6 +136,11 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { + def int bbn = basebuildnumber.trim().toInteger() + return generate(meta, String sourcerevision, String sourcepath, bbn) +} + int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" From f4c573627e3e0a949d04764bf211733892d0c096 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:46:09 +0000 Subject: [PATCH 03/28] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f26806d..e2dc932 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -18,7 +18,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) + adjustedchangecount = ((changecount + basebuildnumber).abs()) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() From db1ebc1959b7bb4818aacc8f79e830cba1431a70 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:50:06 +0000 Subject: [PATCH 04/28] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e2dc932..e28fa2f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,7 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { def int bbn = basebuildnumber.trim().toInteger() - return generate(meta, String sourcerevision, String sourcepath, bbn) + return generate(meta, sourcerevision, sourcepath, bbn) } int incrementrevision(int current, int increment) { From fc6258b158c691c550194a50db86bbd06fb83af4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:55:27 +0000 Subject: [PATCH 05/28] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e28fa2f..4b11cf0 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,7 +136,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } -String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { def int bbn = basebuildnumber.trim().toInteger() return generate(meta, sourcerevision, sourcepath, bbn) } From f7010f5eaf8cb9aeb77ca252f0616b14ceb19b0c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 14:03:46 +0000 Subject: [PATCH 06/28] Set buildstamp --- meta/folder.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 15f77d0..629aa5b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -161,7 +161,7 @@ class BuildStampInfo extends GISJsonSerialised { int revision String buildstamp - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { + BuildStampInfo(script, buildstampprefix = "1.30000.0.", revision = 0, buildstamp = "1.30000.0.0") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision @@ -188,6 +188,12 @@ class BuildStampInfo extends GISJsonSerialised { script.echo("Build stamp now ${buildstamp}") } + void set(String buildstampprefix, int revision) { + buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' + this.buildstampprefix = buildstampprefix + changerevision(revision) + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From f664375cc2bf27ccd89fc672b01d418a4a3c5462 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 14:10:10 +0000 Subject: [PATCH 07/28] Set buildstamp --- meta/folder.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 629aa5b..5ba3485 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -194,6 +194,10 @@ class BuildStampInfo extends GISJsonSerialised { changerevision(revision) } + void set(String buildstampprefix, String revision) { + set(buildstampprefix, revision.toInteger()) + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From 83c89342711ecdc6509d1f5a312990d21b585a4c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 14:21:29 +0000 Subject: [PATCH 08/28] Set buildstamp --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 5ba3485..39de31f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -191,6 +191,7 @@ class BuildStampInfo extends GISJsonSerialised { void set(String buildstampprefix, int revision) { buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' this.buildstampprefix = buildstampprefix + this.buildstamp = buildstampprefix + "0" changerevision(revision) } From fc17af2c5faf83682d17cd9bead0e3f5972c39e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 16:09:25 +0000 Subject: [PATCH 09/28] Handle components that don't have associated source code --- release/do.groovy | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 85753e6..1ded952 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -68,23 +68,25 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } } } } @@ -297,10 +299,15 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" def String buildsha = buildmeta.gitinfo.gitrevision - def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) - def tagadded = sourcegittags(buildsha, srctaglist) - if (tagadded) { - sourcegitpush(dryrunpush) + if (buildsha) { + def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } + else { + summaryappend(",No Src ") } } } From fa1feb73b91d0ac86865e1c8d9bca9d8b0a1f9fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 5 Jan 2019 12:18:09 +0000 Subject: [PATCH 10/28] Automatically add the top level meta item tag to all subcomponents --- release/release.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/release.groovy b/release/release.groovy index 828f1ff..3c13368 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -42,6 +42,9 @@ pipeline { steps { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + // Place the top level build stamp on all + def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" + OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason From 67d92ba13af4f6cf4e3143d83efc72e31f3f3a3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Jan 2019 14:13:42 +0000 Subject: [PATCH 11/28] Make other tags component unique --- release/do.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 1ded952..852f6f3 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -157,14 +157,15 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { return changed } -def ArrayList maketaglist(String buildstamp, String otherreleasetags) { +def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { def ArrayList taglist = new ArrayList() taglist.add(buildstamp) def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - taglist.add(othertagitem.trim()) + componentothertagitem = "${othertagitem}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem.trim()) } } return taglist @@ -288,7 +289,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + ArrayList taglist = maketaglist(buildmeta, buildstamp, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { @@ -300,7 +301,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { - def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamp, otherreleasetags) def tagadded = sourcegittags(buildsha, srctaglist) if (tagadded) { sourcegitpush(dryrunpush) From 94b5ef18591b9a26758c93702221dc51d04ebc75 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Jan 2019 14:25:20 +0000 Subject: [PATCH 12/28] Make other tags component unique; Report more info --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 852f6f3..0e53b7f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -32,7 +32,8 @@ def preconditions() { def setunstable(String reason) { markunstable = true - unstablereason = reason + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" } def summaryappend(String summary) { From f4d8ff7a5315f15a6a91ad4eb2fd10c086bd582b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Jan 2019 14:35:37 +0000 Subject: [PATCH 13/28] Make other tags component unique; Report more info --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0e53b7f..0ea3399 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -165,8 +165,8 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - componentothertagitem = "${othertagitem}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" - taglist.add(componentothertagitem.trim()) + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem) } } return taglist From 329997e9829484c8d350d339fca2bd645c594bf4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 12 Jan 2019 12:35:30 +0000 Subject: [PATCH 14/28] On releasing, release the top level component last - so that we don't end up with partial release info --- meta/folder.groovy | 6 +++++- release/do.groovy | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 39de31f..ff56928 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -658,12 +658,16 @@ class MetaInfo extends GISJsonSerialised { return releasemetacollection } + String componentidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + } + Map releasewalkerunique(int maxdepth, String startpath = "") { Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { - String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + String uniquestring = relatedmetainfo.meta.componentidentifier() releasemetauniquecollection.put(uniquestring, relatedmetainfo) } return releasemetauniquecollection diff --git a/release/do.groovy b/release/do.groovy index 0ea3399..e264cc0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,11 +341,18 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) + def toplevelid = buildmeta.componentidentifier() // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - releasecomponent(buildinfo, otherreleasetags, dryrunpush) + if (toplevelid != key) { + releasecomponent(buildinfo, otherreleasetags, dryrunpush) + } } + // Do the top level last + def toplevelbuildinfo = componentReleaseInfo[toplevelid] + releasecomponent(toplevelbuildinfo, otherreleasetags, dryrunpush) + return [summarystring, markunstable, unstablereason] } From 50db2ba7dfbed6d84b61c4679fb34b92b46066c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 07:58:47 +0000 Subject: [PATCH 15/28] Publishing jobs need to provide the meta depth --- publish/publish.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index 8ed7aa8..a703c9e 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -44,7 +44,6 @@ pipeline { steps { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - MAX_META_DEPTH='2' (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason From 59a74b1f9aceda55d7f8a1d1f6f7dbf65b2fb412 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 10:11:51 +0000 Subject: [PATCH 16/28] Do an integrity check at release time; Print out information about components --- publish/do.groovy | 10 ++++++++-- release/do.groovy | 23 +++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index e280c0c..76dc025 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -107,10 +107,16 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin resetfolders() preconditions() - // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = releasemeta.releasewalkerunique(md) + // Walk the meta information to a max depth, looking for artifacts to publish + (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) // For each component + echo "Components in publish:" + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, softwarerelease) diff --git a/release/do.groovy b/release/do.groovy index e264cc0..cbecc08 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -338,8 +338,27 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String resetfolders() preconditions() - // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = buildmeta.releasewalkerunique(md) + // Walk the meta information to a max depth, looking for components to release + (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) + + + echo "Components in release:" + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + + echo "Version unique components in release:" + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + + if (DO_INTEGRITY_CHECK == "true") { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique()) { + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" + } + } def toplevelid = buildmeta.componentidentifier() // For each component From d7834ac5895432b8fdf8ea4e855ec7a70d7d3dac Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 10:36:00 +0000 Subject: [PATCH 17/28] Do an integrity check at release time; Print out information about components --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index cbecc08..a8f9e49 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -339,7 +339,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String preconditions() // Walk the meta information to a max depth, looking for components to release - (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) + (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) echo "Components in release:" From 4b720911b4be5e97c1f9ae0ba5a5088eafaf4bdd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 12:59:29 +0000 Subject: [PATCH 18/28] Do an integrity check at release time; Print out information about components --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index a8f9e49..d1952f0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,7 +341,6 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) - echo "Components in release:" for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] From 4ad0007dceb8155e7d6988c26e7e15e21aaae9f0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:14:57 +0000 Subject: [PATCH 19/28] Do an integrity check at release time; Print out information about components --- publish/do.groovy | 11 ++--------- release/do.groovy | 14 +------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 76dc025..907f328 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -110,15 +110,8 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // Walk the meta information to a max depth, looking for artifacts to publish (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) - // For each component - echo "Components in publish:" - for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" - } - - for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] recallcomponent(releaseinfo, softwarerelease) } stripgithistory() diff --git a/release/do.groovy b/release/do.groovy index d1952f0..b2577fb 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,20 +341,8 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) - echo "Components in release:" - for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" - } - - echo "Version unique components in release:" - for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" - } - if (DO_INTEGRITY_CHECK == "true") { - if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique()) { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" } } From ce93cdb372560f2313802310f84442ac3e168a8a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:40:33 +0000 Subject: [PATCH 20/28] Do an integrity check at release time; Print out information about components --- meta/folder.groovy | 25 ++++++++++++++++++++++--- publish/do.groovy | 8 +++++++- release/do.groovy | 17 ++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ff56928..2ef8e22 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -662,17 +662,36 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname } + String componentversionuniqueidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp + } + Map releasewalkerunique(int maxdepth, String startpath = "") { - Map releasemetauniquecollection = new LinkedHashMap() - ArrayList releasemetacollection = new ArrayList() + def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + + def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { - String uniquestring = relatedmetainfo.meta.componentidentifier() + def String uniquestring = relatedmetainfo.meta.componentidentifier() releasemetauniquecollection.put(uniquestring, relatedmetainfo) } + return releasemetauniquecollection } + Map releasewalkerversionunique(int maxdepth, String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + + def Map releasemetaversionuniquecollection = new LinkedHashMap() + for (relatedmetainfo in releasemetacollection) { + def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() + releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + } + + return releasemetaversionuniquecollection + } + MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } diff --git a/publish/do.groovy b/publish/do.groovy index 907f328..a88f018 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -108,7 +108,13 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin preconditions() // Walk the meta information to a max depth, looking for artifacts to publish - (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) + + echo "Version unique components:" + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] diff --git a/release/do.groovy b/release/do.groovy index b2577fb..172861b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -339,7 +339,22 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String preconditions() // Walk the meta information to a max depth, looking for components to release - (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + + script.echo "Unique components:" + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + script.echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) + + echo "Version unique components:" + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { From 20e86747ef8b0a174c34e15f284e647ecbbc87de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:41:54 +0000 Subject: [PATCH 21/28] Do an integrity check at release time; Print out information about components --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 172861b..0c070d1 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,10 +341,10 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release def componentReleaseInfo = buildmeta.releasewalkerunique(md) - script.echo "Unique components:" + echo "Unique components:" for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - script.echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" } // Walk the meta information to a max depth, looking for components to release From 3567488f00d39277ee8a22736289b62db34b7ae3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:51:17 +0000 Subject: [PATCH 22/28] Do an integrity check at release time; Print out information about components --- publish/do.groovy | 5 +++-- release/do.groovy | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index a88f018..570cb5e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -111,10 +111,11 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) echo "Version unique components:" + def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + versionuniquecomponent += "${key}\n" } + echo versionuniquecomponent for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] diff --git a/release/do.groovy b/release/do.groovy index 0c070d1..2326ad5 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -340,21 +340,21 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release def componentReleaseInfo = buildmeta.releasewalkerunique(md) - echo "Unique components:" + def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + versioncomponent += "${key}\n" } + echo versioncomponent // Walk the meta information to a max depth, looking for components to release def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) - echo "Version unique components:" + def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + versionuniquecomponent += "${key}\n" } + echo versionuniquecomponent if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { From 42df36426c1496d2e67162e9519dfdab542dea0b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 21 Jan 2019 15:15:27 +0000 Subject: [PATCH 23/28] Handle missing jq command --- differences/json.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/differences/json.groovy b/differences/json.groovy index 47d9cc5..19b7cf6 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -3,6 +3,8 @@ def comparefiles(importedjsonfile, currentjsonfile) { env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ + which jq + if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") """ From 4fd07d22d652a396fab26ef6f468ad509786faf8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Jan 2019 15:32:00 +0000 Subject: [PATCH 24/28] Handle the case where rebuilt binaries change even though the original source has not changed and the version number has not changed - Tags now have the build number appended to them so that there is no tag clash during the release process --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2326ad5..79bafd5 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -270,7 +270,7 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp + def String buildstamp = "${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() @@ -298,7 +298,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" + def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamp}" def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { From ca173c1f1310b1e743e626b4b2784b3eef3dbef2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Jan 2019 15:53:13 +0000 Subject: [PATCH 25/28] Handle the case where rebuilt binaries change even though the original source has not changed and the version number has not changed - Tags now have the build number appended to them so that there is no tag clash during the release process - change the meta build tags too --- release/do.groovy | 11 ++++++----- release/release.groovy | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 79bafd5..1a785b3 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -270,7 +270,8 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = "${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" + def String buildstamp = buildmeta.buildstampinfo.buildstamp + def String buildstamptag = "${buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() @@ -280,7 +281,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def boolean changed = checkforchange(buildmeta, previousreleasemeta) if (changed) { - releasegitcommit(buildstamp) + releasegitcommit(buildstamptag) } else { releasegitreset() @@ -290,7 +291,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - ArrayList taglist = maketaglist(buildmeta, buildstamp, otherreleasetags) + ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { @@ -298,11 +299,11 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamp}" + def String srcbuildstamptag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamptag}" def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { - def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamp, otherreleasetags) + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) def tagadded = sourcegittags(buildsha, srctaglist) if (tagadded) { sourcegitpush(dryrunpush) diff --git a/release/release.groovy b/release/release.groovy index 3c13368..b0530be 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,7 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" + def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { From 5c5bc9a97804dcf418a87eff5bb187fdc581239a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 27 Jan 2019 11:11:17 +0000 Subject: [PATCH 26/28] Adjust publish script to handle buildstamps appended with build number --- publish/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 570cb5e..b188165 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -44,7 +44,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def releasemeta = releaseinfo.meta def String componentURL = releasemeta.releaseinfo.releaserepoURL - def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo def ArrayList publishablePaths = new ArrayList() if (publishableitems.size() > 0) { @@ -78,11 +78,11 @@ def publish(String customer) { def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}_${releasemeta.jenkinsbuildinfo.buildnumber}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname} ${releasemeta.jenkinsbuildinfo.buildnumber}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) From 91f0f07e8d4ca0aa3fea5538eb1904e2f92b44ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 27 Jan 2019 11:27:09 +0000 Subject: [PATCH 27/28] Adjust publish script to handle buildstamps appended with build number - The output folders and summary don't need to be keyed by build number --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b188165..7844289 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -78,11 +78,11 @@ def publish(String customer) { def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}_${releasemeta.jenkinsbuildinfo.buildnumber}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname} ${releasemeta.jenkinsbuildinfo.buildnumber}: ") + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) From fb25a6b30b16515306aaa7032dcf94484d82e53c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 27 Jan 2019 12:21:29 +0000 Subject: [PATCH 28/28] When generating revision numbers, take into account negative basebuildnumbers --- versionnumbering/generate.groovy | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 4b11cf0..4761174 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -4,6 +4,12 @@ def int maxversionnumber() { return MAXVERSIONNUMBER } +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" + return newrevision +} + def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount def String gitrevision @@ -18,7 +24,9 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" - adjustedchangecount = ((changecount + basebuildnumber).abs()) % (maxversionnumber() + 1) + def int mvnp1 = maxversionnumber() + 1 + basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber + adjustedchangecount = incrementrevision(changecount, basebuildnumber) echo "Adjusted change count: ${adjustedchangecount}" if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -141,10 +149,4 @@ String generate(meta, String sourcerevision, String sourcepath, String basebuild return generate(meta, sourcerevision, sourcepath, bbn) } -int incrementrevision(int current, int increment) { - def int newrevision = (current + increment) % (maxversionnumber() + 1) - echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" - return newrevision -} - return this; \ No newline at end of file