From 26650ee254403cc2d6a850be1241b9000e124530 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:27:20 +0100 Subject: [PATCH 01/10] Add skeletal meta differences script files --- differences/dometahierarchy.groovy | 10 ++++ differences/metahierarchy.groovy | 92 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 differences/dometahierarchy.groovy create mode 100644 differences/metahierarchy.groovy diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy new file mode 100644 index 0000000..65e6cf4 --- /dev/null +++ b/differences/dometahierarchy.groovy @@ -0,0 +1,10 @@ + +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String summaryfolder, int maxdepth) { + def String summarystring = "No summary" + def boolean markunstable = true + def unstablereason = "No compare implementation yet" + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy new file mode 100644 index 0000000..3ea9dbc --- /dev/null +++ b/differences/metahierarchy.groovy @@ -0,0 +1,92 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + SUMMARYDIR = "${WORKSPACE}/summary" + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + "${WORKSPACE}/early", + "${WORKSPACE}/later", + "${SUMMARYDIR}", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout top level meta') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = 'git@hydra:gis/newsw.git' + + EARLYREVISION = "${RELEASE_EARLY_REVISION}".trim() + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) + + LATERREVISION = "${RELEASE_LATER_REVISION}".trim() + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) + } + } + } + stage('Diff components') { + steps { + script { + def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, SUMMARYDIR, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From f70406e72ba6ed5188134541454dd78a8f9561e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:35:45 +0100 Subject: [PATCH 02/10] Add skeletal meta differences script files --- differences/metahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 3ea9dbc..d82fcc0 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -29,14 +29,14 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - SOURCEURL = 'git@hydra:gis/newsw.git' + SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - EARLYREVISION = "${RELEASE_EARLY_REVISION}".trim() + EARLYREVISION = RELEASE_EARLY_REVISION.trim() EARLYROOTDIR = 'early' EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) - LATERREVISION = "${RELEASE_LATER_REVISION}".trim() + LATERREVISION = RELEASE_LATER_REVISION.trim() LATERROOTDIR = 'later' LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) From 36224b7c865a8218da05cc410ad96e2eddf50b86 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:48:25 +0100 Subject: [PATCH 03/10] Add skeletal meta differences script files --- differences/dometahierarchy.groovy | 6 +++++- differences/metahierarchy.groovy | 2 +- release/do.groovy | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 65e6cf4..1d6885f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,5 +1,9 @@ -def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String summaryfolder, int maxdepth) { +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def int md = maxdepth.toInteger() + + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index d82fcc0..cd6738d 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -47,7 +47,7 @@ pipeline { steps { script { def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") - (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, SUMMARYDIR, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' diff --git a/release/do.groovy b/release/do.groovy index b3d54c7..738c04a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -365,7 +365,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri } def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { - int md = maxdepth.toInteger() + def int md = maxdepth.toInteger() resetfolders() preconditions() From 7e5c624d72bec8116849ac59e57cb75272981407 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 May 2019 11:02:28 +0100 Subject: [PATCH 04/10] Strip *@tmp folders from published folder --- versioncontrol/git.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8886076..534084d 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -323,6 +323,7 @@ def stripgithistory(String rootfolder) { find . -name ".gitignore" | xargs -iXX rm -rf XX find . -name ".gitmodules" | xargs -iXX rm -rf XX find . -iname ".gitattributes" | xargs -iXX rm -rf XX + find . -iname "*@tmp" | xargs -iXX rm -rf XX """ } else { echo "Not supporting git stripping in Windows at the moment" From 8961f250b17a89620fa0ea35b796d664512fdfb8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 19:39:52 +0100 Subject: [PATCH 05/10] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 23 ++++++++++++++++++++++- publish/do.groovy | 13 +++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 34707bc..9620e8a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,7 +42,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -76,6 +76,27 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(destpath) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..6207380 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,6 +148,19 @@ def convertitemstopublish(String itemstopublishstring) { return paths } +def convertstringitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { From e418c593be9bb8ba865142321491b846567c3a7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:04:39 +0100 Subject: [PATCH 06/10] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 9620e8a..d85818f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) From a5a2603e27dfd1e94c84b134a06d07d0dfc5f8f6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:25:53 +0100 Subject: [PATCH 07/10] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- publish/do.groovy | 13 ------------- versioncontrol/git.groovy | 13 +++++++++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index d85818f..764618c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) diff --git a/publish/do.groovy b/publish/do.groovy index 6207380..1599ca8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,19 +148,6 @@ def convertitemstopublish(String itemstopublishstring) { return paths } -def convertstringitemstopublish(String itemstopublishstring) { - def List paths = [] - - def String[] items = itemstopublishstring.split(',') - def String allpaths = "" - for(String item : items) { - paths.add(item) - allpaths += "${path}\n" - } - echo "Selected item to publish:\n${allpaths}" - return paths -} - // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 534084d..818734a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -246,6 +246,19 @@ def gitsparserestore(String targetDir) { } } +def List convertSparseItemsString(String itemstopublish) { + def List paths = [] + + def String[] items = itemstopublish.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) From f1cb5075a2794e74a09d58c5f50a118f9813eeff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:33:55 +0100 Subject: [PATCH 08/10] MCS-425 - Perform a sparse checkout of items to import from a release repo --- versioncontrol/git.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 818734a..811d072 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -247,16 +247,16 @@ def gitsparserestore(String targetDir) { } def List convertSparseItemsString(String itemstopublish) { - def List paths = [] + def List sparsepaths = [] def String[] items = itemstopublish.split(',') def String allpaths = "" for(String item : items) { - paths.add(item) - allpaths += "${path}\n" + sparsepaths.add(item) + allpaths += "${item}\n" } echo "Selected item to publish:\n${allpaths}" - return paths + return sparsepaths } def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { From 2c5caba8ec8d123c2e4f852a7c33baafce6723e2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 13:30:15 +0100 Subject: [PATCH 09/10] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- meta/folder.groovy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 764618c..fd43379 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -68,7 +68,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) diff --git a/meta/folder.groovy b/meta/folder.groovy index b16db5d..f8c033b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -491,6 +491,7 @@ class MetaInfo extends GISJsonSerialised { } void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") releaseinfo.markreleased() for (associated in associatedmeta) { associated.meta.releaseinfo.markreleased() From 633a8dc8cb9bd107300c238a539cf97a56b5bcec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 14:08:13 +0100 Subject: [PATCH 10/10] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index fd43379..09eb337 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -89,7 +89,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))