From 74ad66b19964a07c2b14ea0a727234db1446dd49 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 10 Dec 2021 17:10:24 +0000 Subject: [PATCH 001/102] Start writing format version into meta data file --- meta/folder.groovy | 49 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d67389f..70d024a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -20,12 +20,14 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' String productname String subcomponentname String variant ProductInfo(script, productname = "", subcomponentname = "", variant = "") { super(script) + this.format = FORMAT this.productname = productname this.subcomponentname = subcomponentname this.variant = variant @@ -77,6 +79,7 @@ class ProductInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] variant = rawmeta['variant'] ?: "" @@ -84,6 +87,7 @@ class ProductInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'productname' : productname, 'subcomponentname' : subcomponentname, 'variant' : variant, @@ -93,6 +97,7 @@ class ProductInfo extends GISJsonSerialised { } class GitInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' private String sourcepath private String sourcerevision private String gitrevision @@ -101,6 +106,7 @@ class GitInfo extends GISJsonSerialised { GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { super(script) + this.format = FORMAT this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -134,6 +140,7 @@ class GitInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] sourcepath = rawmeta['sourcepath'] sourcerevision = rawmeta['sourcerevision'] gitrevision = rawmeta['gitrevision'] @@ -143,6 +150,7 @@ class GitInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, 'gitrevision' : gitrevision, @@ -154,12 +162,14 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildParameter extends GISJsonSerialised { + private static String FORMAT = '0.1' private String name private String valuetype private String value JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { super(script) + this.format = FORMAT this.name = name this.valuetype = valuetype this.value = value @@ -172,6 +182,7 @@ class JenkinsBuildParameter extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] name = rawmeta['name'] valuetype = rawmeta['valuetype'] value = rawmeta['value'] @@ -179,6 +190,7 @@ class JenkinsBuildParameter extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'name' : name, 'valuetype' : valuetype, 'value' : value, @@ -188,6 +200,7 @@ class JenkinsBuildParameter extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' String buildjobname int buildnumber private String buildmachine @@ -197,6 +210,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "", boolean isfullbuild = true) { super(script) + this.format = FORMAT this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine @@ -228,6 +242,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] buildjobname = rawmeta['buildjobname'] buildnumber = rawmeta['buildnumber'] buildmachine = rawmeta['buildmachine'] @@ -253,6 +268,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, 'buildmachine' : buildmachine, @@ -265,12 +281,14 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' String buildstampprefix int revision String buildstamp BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) + this.format = FORMAT this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp @@ -333,6 +351,7 @@ class BuildStampInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] @@ -340,6 +359,7 @@ class BuildStampInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, @@ -349,12 +369,14 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' String releaserepoURL ArrayList releaseitems boolean released ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { super(script) + this.format = FORMAT this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() this.released = released @@ -382,6 +404,7 @@ class ReleaseInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] releaserepoURL = rawmeta['releaserepoURL'] releaseitems = new ArrayList() for (rawreleaseitem in rawmeta['releaseitems']) { @@ -392,6 +415,7 @@ class ReleaseInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'releaserepoURL' : releaserepoURL, 'releaseitems' : releaseitemssserialised(), 'released' : released @@ -401,6 +425,7 @@ class ReleaseInfo extends GISJsonSerialised { } class ConfigurationInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' private String configurationname private String configurationURL private String configurationversion @@ -408,6 +433,7 @@ class ConfigurationInfo extends GISJsonSerialised { ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { super(script) + this.format = FORMAT this.configurationname = configurationname this.configurationURL = configurationURL this.configurationversion = configurationversion @@ -415,6 +441,7 @@ class ConfigurationInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] configurationname = rawmeta['configurationname'] configurationURL = rawmeta['configurationURL'] configurationversion = rawmeta['configurationversion'] @@ -423,6 +450,7 @@ class ConfigurationInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'configurationname' : configurationname, 'configurationURL' : configurationURL, 'configurationversion' : configurationversion, @@ -457,11 +485,13 @@ class ChecksumPair extends GISJsonSerialised { } class ChecksumSet extends GISJsonSerialised { + private static String FORMAT = '0.1' private String artifactpath private ArrayList checksumpairs ChecksumSet(script, artifactpath = "") { super(script) + this.format = FORMAT this.artifactpath = artifactpath checksumpairs = new ArrayList() } @@ -481,6 +511,7 @@ class ChecksumSet extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] artifactpath = rawmeta['artifactpath'] checksumpairs = new ArrayList() for (rawchecksumpair in rawmeta['checksumpairs']) { @@ -492,6 +523,7 @@ class ChecksumSet extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'artifactpath' : artifactpath, 'checksumpairs' : checksumpairsserialised(), ] @@ -500,10 +532,12 @@ class ChecksumSet extends GISJsonSerialised { } class ChecksumInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' private ArrayList checksumsets ChecksumInfo(script) { super(script) + this.format = FORMAT checksumsets = new ArrayList() } @@ -522,6 +556,7 @@ class ChecksumInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] checksumsets = new ArrayList() for (rawchecksumset in rawmeta['checksumsets']) { def cs = new ChecksumSet(script) @@ -532,6 +567,7 @@ class ChecksumInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'checksumsets' : checksumsetsserialised(), ] return data @@ -539,19 +575,23 @@ class ChecksumInfo extends GISJsonSerialised { } class PublishableInfo extends GISJsonSerialised { + private static String FORMAT = '0.1' private String filepath PublishableInfo(script, String filepath = "") { super(script) + this.format = FORMAT this.filepath = filepath } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] filepath = rawmeta['filepath'] } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'filepath' : filepath, ] return data @@ -559,9 +599,9 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private static String FORMAT = "0.1" - private static String METAFOLDERNAME = "meta" - private static String METAFILENAME = "info.json" + private static String FORMAT = '0.1' + private static String METAFOLDERNAME = 'meta' + private static String METAFILENAME = 'info.json' private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { @@ -570,11 +610,13 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { super(script) + this.format = FORMAT this.offsetpath = offsetpath this.meta = meta } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] offsetpath = rawmeta['offsetpath'] meta = new MetaInfo(script) meta.deserialiseJson(rawmeta['meta']) @@ -582,6 +624,7 @@ class MetaInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'offsetpath' : offsetpath, 'meta' : meta.serialisationdata(), ] From c66149252d3ceb2370c99ce34f10ed61c9ee2bfe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 10 Dec 2021 17:26:02 +0000 Subject: [PATCH 002/102] Start writing format version into meta data file --- meta/folder.groovy | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 70d024a..9b4452a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -20,14 +20,13 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' String productname String subcomponentname String variant ProductInfo(script, productname = "", subcomponentname = "", variant = "") { super(script) - this.format = FORMAT + this.format = '0.1' this.productname = productname this.subcomponentname = subcomponentname this.variant = variant @@ -97,7 +96,6 @@ class ProductInfo extends GISJsonSerialised { } class GitInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' private String sourcepath private String sourcerevision private String gitrevision @@ -106,7 +104,7 @@ class GitInfo extends GISJsonSerialised { GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { super(script) - this.format = FORMAT + this.format = '0.1' this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -162,14 +160,13 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildParameter extends GISJsonSerialised { - private static String FORMAT = '0.1' private String name private String valuetype private String value JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { super(script) - this.format = FORMAT + this.format = '0.1' this.name = name this.valuetype = valuetype this.value = value @@ -200,7 +197,6 @@ class JenkinsBuildParameter extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' String buildjobname int buildnumber private String buildmachine @@ -210,7 +206,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "", boolean isfullbuild = true) { super(script) - this.format = FORMAT + this.format = '0.1' this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine @@ -281,14 +277,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' String buildstampprefix int revision String buildstamp BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) - this.format = FORMAT + this.format = '0.1' this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp @@ -369,14 +364,13 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' String releaserepoURL ArrayList releaseitems boolean released ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { super(script) - this.format = FORMAT + this.format = '0.1' this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() this.released = released @@ -425,7 +419,6 @@ class ReleaseInfo extends GISJsonSerialised { } class ConfigurationInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' private String configurationname private String configurationURL private String configurationversion @@ -433,7 +426,7 @@ class ConfigurationInfo extends GISJsonSerialised { ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { super(script) - this.format = FORMAT + this.format = '0.1' this.configurationname = configurationname this.configurationURL = configurationURL this.configurationversion = configurationversion @@ -485,13 +478,12 @@ class ChecksumPair extends GISJsonSerialised { } class ChecksumSet extends GISJsonSerialised { - private static String FORMAT = '0.1' private String artifactpath private ArrayList checksumpairs ChecksumSet(script, artifactpath = "") { super(script) - this.format = FORMAT + this.format = '0.1' this.artifactpath = artifactpath checksumpairs = new ArrayList() } @@ -532,12 +524,11 @@ class ChecksumSet extends GISJsonSerialised { } class ChecksumInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' private ArrayList checksumsets ChecksumInfo(script) { super(script) - this.format = FORMAT + this.format = '0.1' checksumsets = new ArrayList() } @@ -575,12 +566,11 @@ class ChecksumInfo extends GISJsonSerialised { } class PublishableInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' private String filepath PublishableInfo(script, String filepath = "") { super(script) - this.format = FORMAT + this.format = '0.1' this.filepath = filepath } @@ -599,7 +589,6 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private static String FORMAT = '0.1' private static String METAFOLDERNAME = 'meta' private static String METAFILENAME = 'info.json' private static String METAARTIFACTS = "${METAFOLDERNAME}/**" @@ -610,7 +599,7 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { super(script) - this.format = FORMAT + this.format = '0.1' this.offsetpath = offsetpath this.meta = meta } @@ -632,7 +621,7 @@ class MetaInfo extends GISJsonSerialised { } } - private String format = FORMAT + private String format = '0.1' ProductInfo productinfo GitInfo gitinfo JenkinsBuildInfo jenkinsbuildinfo @@ -720,7 +709,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - if (rawmeta['format'] != FORMAT) { + if (rawmeta['format'] != '0.1') { script.error("Unsupported meta format found at ${filepath}") return } @@ -969,9 +958,9 @@ class MetaInfo extends GISJsonSerialised { } MetaInfo metareset(script) { - echo "Resetting meta folder" def mi = new MetaInfo(script) + echo "Resetting meta folder" dir(mi.folder()) { deleteDir() } From df492c865a00124636569d39180756641abc0ff4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 10 Dec 2021 17:31:21 +0000 Subject: [PATCH 003/102] Start writing format version into meta data file --- meta/folder.groovy | 53 ++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9b4452a..806b6ff 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -20,9 +20,10 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - String productname - String subcomponentname - String variant + private String format + private String productname + private String subcomponentname + private String variant ProductInfo(script, productname = "", subcomponentname = "", variant = "") { super(script) @@ -96,6 +97,7 @@ class ProductInfo extends GISJsonSerialised { } class GitInfo extends GISJsonSerialised { + private String format private String sourcepath private String sourcerevision private String gitrevision @@ -160,6 +162,7 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildParameter extends GISJsonSerialised { + private String format private String name private String valuetype private String value @@ -197,12 +200,13 @@ class JenkinsBuildParameter extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { - String buildjobname - int buildnumber + private String format + private String buildjobname + private int buildnumber private String buildmachine private String builddate - ArrayList parameters - boolean isfullbuild + private ArrayList parameters + private boolean isfullbuild JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "", boolean isfullbuild = true) { super(script) @@ -277,9 +281,10 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - String buildstampprefix - int revision - String buildstamp + private String format + private String buildstampprefix + private int revision + private String buildstamp BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) @@ -364,9 +369,10 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - String releaserepoURL - ArrayList releaseitems - boolean released + private String format + private String releaserepoURL + private ArrayList releaseitems + private boolean released ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { super(script) @@ -419,6 +425,7 @@ class ReleaseInfo extends GISJsonSerialised { } class ConfigurationInfo extends GISJsonSerialised { + private String format private String configurationname private String configurationURL private String configurationversion @@ -478,6 +485,7 @@ class ChecksumPair extends GISJsonSerialised { } class ChecksumSet extends GISJsonSerialised { + private String format private String artifactpath private ArrayList checksumpairs @@ -524,6 +532,7 @@ class ChecksumSet extends GISJsonSerialised { } class ChecksumInfo extends GISJsonSerialised { + private String format private ArrayList checksumsets ChecksumInfo(script) { @@ -566,6 +575,7 @@ class ChecksumInfo extends GISJsonSerialised { } class PublishableInfo extends GISJsonSerialised { + private String format private String filepath PublishableInfo(script, String filepath = "") { @@ -594,8 +604,9 @@ class MetaInfo extends GISJsonSerialised { private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { - String offsetpath - MetaInfo meta + private String format + private String offsetpath + private MetaInfo meta RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { super(script) @@ -622,12 +633,12 @@ class MetaInfo extends GISJsonSerialised { } private String format = '0.1' - ProductInfo productinfo - GitInfo gitinfo - JenkinsBuildInfo jenkinsbuildinfo - BuildStampInfo buildstampinfo - ReleaseInfo releaseinfo - ChecksumInfo checksuminfo + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + private ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo private LinkedHashMap custominfo From 0701945bab8ac31fdc2080f81e5c89e63a8f6c2a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 10 Dec 2021 17:41:14 +0000 Subject: [PATCH 004/102] Start writing format version into meta data file --- meta/folder.groovy | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 806b6ff..4efb0d2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -162,14 +162,12 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildParameter extends GISJsonSerialised { - private String format private String name private String valuetype private String value JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { super(script) - this.format = '0.1' this.name = name this.valuetype = valuetype this.value = value @@ -182,7 +180,6 @@ class JenkinsBuildParameter extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - //format = rawmeta['format'] name = rawmeta['name'] valuetype = rawmeta['valuetype'] value = rawmeta['value'] @@ -190,7 +187,6 @@ class JenkinsBuildParameter extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ - 'format' : format, 'name' : name, 'valuetype' : valuetype, 'value' : value, @@ -425,7 +421,6 @@ class ReleaseInfo extends GISJsonSerialised { } class ConfigurationInfo extends GISJsonSerialised { - private String format private String configurationname private String configurationURL private String configurationversion @@ -433,7 +428,6 @@ class ConfigurationInfo extends GISJsonSerialised { ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { super(script) - this.format = '0.1' this.configurationname = configurationname this.configurationURL = configurationURL this.configurationversion = configurationversion @@ -441,7 +435,6 @@ class ConfigurationInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - //format = rawmeta['format'] configurationname = rawmeta['configurationname'] configurationURL = rawmeta['configurationURL'] configurationversion = rawmeta['configurationversion'] @@ -450,7 +443,6 @@ class ConfigurationInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ - 'format' : format, 'configurationname' : configurationname, 'configurationURL' : configurationURL, 'configurationversion' : configurationversion, @@ -532,12 +524,10 @@ class ChecksumSet extends GISJsonSerialised { } class ChecksumInfo extends GISJsonSerialised { - private String format private ArrayList checksumsets ChecksumInfo(script) { super(script) - this.format = '0.1' checksumsets = new ArrayList() } @@ -556,7 +546,6 @@ class ChecksumInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - //format = rawmeta['format'] checksumsets = new ArrayList() for (rawchecksumset in rawmeta['checksumsets']) { def cs = new ChecksumSet(script) @@ -567,7 +556,6 @@ class ChecksumInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ - 'format' : format, 'checksumsets' : checksumsetsserialised(), ] return data From 19d7103f315d9ce1aa5348ddc907231dad6668c9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 10 Dec 2021 18:20:14 +0000 Subject: [PATCH 005/102] Don't serialise associated meta data --- meta/folder.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 4efb0d2..fe8b565 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -593,12 +593,14 @@ class MetaInfo extends GISJsonSerialised { class RelatedMetaInfo extends GISJsonSerialised { private String format + private String fullpath private String offsetpath private MetaInfo meta - RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + RelatedMetaInfo(script, fullpath = "", offsetpath = "", MetaInfo meta = null) { super(script) - this.format = '0.1' + this.format = '0.2' + this.fullpath = fullpath this.offsetpath = offsetpath this.meta = meta } @@ -614,7 +616,7 @@ class MetaInfo extends GISJsonSerialised { def data = [ 'format' : format, 'offsetpath' : offsetpath, - 'meta' : meta.serialisationdata(), + //'meta' : meta.serialisationdata(), ] return data } @@ -779,13 +781,14 @@ class MetaInfo extends GISJsonSerialised { def String adjustedpath = relativize(destpath) def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { + script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) } else { script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, adjustedpath, mi) if (appendtoparentmeta) { associatedmeta.add(rmi) } From dfcbbf1fa6fc9166754dedce9398c24867a9947f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 16:02:22 +0000 Subject: [PATCH 006/102] Record the import path of meta data --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index fe8b565..d3b17c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -593,14 +593,14 @@ class MetaInfo extends GISJsonSerialised { class RelatedMetaInfo extends GISJsonSerialised { private String format - private String fullpath + private String importpath private String offsetpath private MetaInfo meta - RelatedMetaInfo(script, fullpath = "", offsetpath = "", MetaInfo meta = null) { + RelatedMetaInfo(script, importpath = "", offsetpath = "", MetaInfo meta = null) { super(script) this.format = '0.2' - this.fullpath = fullpath + this.importpath = importpath this.offsetpath = offsetpath this.meta = meta } @@ -788,7 +788,7 @@ class MetaInfo extends GISJsonSerialised { mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, adjustedpath, mi) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, importedmetafile, adjustedpath, mi) if (appendtoparentmeta) { associatedmeta.add(rmi) } From f98f9e9565dc0d5e4e486b105c0903ebf9e10cc2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 16:30:20 +0000 Subject: [PATCH 007/102] Relay input meta files to output meta folders --- artifacts/archiver.groovy | 3 ++- meta/folder.groovy | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy index d7b69d0..0ac3a72 100644 --- a/artifacts/archiver.groovy +++ b/artifacts/archiver.groovy @@ -6,7 +6,8 @@ enum Store { this.JENKINS = Store.JENKINS this.ARTIFACTSERVER = Store.JENKINS -def archive(artifactlist, Store store = JENKINS ) { +def archive(meta, artifactlist, Store store = JENKINS ) { + meta.copyrelatedmeta() if (store == JENKINS) { archiveArtifacts(artifactlist) } diff --git a/meta/folder.groovy b/meta/folder.groovy index d3b17c4..d1cec10 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -795,6 +795,18 @@ class MetaInfo extends GISJsonSerialised { return mi } + + def copyrelatedmeta() { + for (relatedmetainfo in associatedmeta) { + def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" + script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") + fileOperations([ + folderCreateOperation(folderPath: destpath), + folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) + ]) + } + } LinkedHashMap serialisationdata() { def data = [ From e466480d24574ba30159a5ccff0682e0fa99dcad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 16:37:06 +0000 Subject: [PATCH 008/102] Relay input meta files to output meta folders --- artifacts/archiver.groovy | 1 + meta/folder.groovy | 1 + 2 files changed, 2 insertions(+) diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy index 0ac3a72..5497106 100644 --- a/artifacts/archiver.groovy +++ b/artifacts/archiver.groovy @@ -7,6 +7,7 @@ this.JENKINS = Store.JENKINS this.ARTIFACTSERVER = Store.JENKINS def archive(meta, artifactlist, Store store = JENKINS ) { + echo "Archiving artifacts to store: ${store}" meta.copyrelatedmeta() if (store == JENKINS) { archiveArtifacts(artifactlist) diff --git a/meta/folder.groovy b/meta/folder.groovy index d1cec10..c0c3620 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -797,6 +797,7 @@ class MetaInfo extends GISJsonSerialised { } def copyrelatedmeta() { + script.echo("Copying related input meta data to output meta") for (relatedmetainfo in associatedmeta) { def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" From 450f38e6fe62e9e8928bbc81ddd4b9df74915958 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 16:58:20 +0000 Subject: [PATCH 009/102] Relay input meta files to output meta folders --- artifacts/archiver.groovy | 2 +- meta/folder.groovy | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy index 5497106..78bee55 100644 --- a/artifacts/archiver.groovy +++ b/artifacts/archiver.groovy @@ -10,7 +10,7 @@ def archive(meta, artifactlist, Store store = JENKINS ) { echo "Archiving artifacts to store: ${store}" meta.copyrelatedmeta() if (store == JENKINS) { - archiveArtifacts(artifactlist) + archiveArtifacts(artifacts: artifactlist) } } diff --git a/meta/folder.groovy b/meta/folder.groovy index c0c3620..efd48da 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -802,9 +802,9 @@ class MetaInfo extends GISJsonSerialised { def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") - fileOperations([ - folderCreateOperation(folderPath: destpath), - folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) + script.fileOperations([ + script.folderCreateOperation(folderPath: destpath), + script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) ]) } } From b397a0722b4ed9694e438edc9fdf789c8af158f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 19:52:11 +0000 Subject: [PATCH 010/102] Remove python checkout when it is not used --- packaging/debian.groovy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index c0a4dbb..4f779bf 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -64,14 +64,12 @@ def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, S return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) } -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs) { try { sh """#!/bin/bash # Exit if any command fails set -xe - echo "${password}" | sudo -S echo - cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" sudo chown -v -R root: "${packagedefs.DEBPATH}" @@ -97,8 +95,6 @@ def buildPackage(String packageout, String uniquepackageout, DebRootDefinition p catch (Exception e) { echo "Exception occurred" sh """#!/bin/bash -v - echo "${password}" | sudo -S echo - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" exit 1 From 36b9f2e187f087b4a85c149b944d4ab3bced7e08 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 20:31:37 +0000 Subject: [PATCH 011/102] Use credential agent to supply password --- packaging/debian.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 4f779bf..c0a4dbb 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -64,12 +64,14 @@ def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, S return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) } -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs) { +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { try { sh """#!/bin/bash # Exit if any command fails set -xe + echo "${password}" | sudo -S echo + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" sudo chown -v -R root: "${packagedefs.DEBPATH}" @@ -95,6 +97,8 @@ def buildPackage(String packageout, String uniquepackageout, DebRootDefinition p catch (Exception e) { echo "Exception occurred" sh """#!/bin/bash -v + echo "${password}" | sudo -S echo + sudo chown -R ${USER}: "${packagedefs.DEBPATH}" exit 1 From c7ff7598600835012ffec28f2bfebe46aa8ac518 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Dec 2021 20:58:52 +0000 Subject: [PATCH 012/102] Use credential agent to supply password --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index efd48da..5edde8f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -608,8 +608,8 @@ class MetaInfo extends GISJsonSerialised { void deserialiseJson(rawmeta) { //format = rawmeta['format'] offsetpath = rawmeta['offsetpath'] - meta = new MetaInfo(script) - meta.deserialiseJson(rawmeta['meta']) + //meta = new MetaInfo(script) + //meta.deserialiseJson(rawmeta['meta']) } LinkedHashMap serialisationdata() { @@ -710,7 +710,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - if (rawmeta['format'] != '0.1') { + if (rawmeta['format'] != '0.1' ) { script.error("Unsupported meta format found at ${filepath}") return } From c39aac83b5125a3364d0af8de156b0e5ecf509ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Dec 2021 17:14:51 +0000 Subject: [PATCH 013/102] Use credential manager to perform sudo operations --- packaging/debian.groovy | 64 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index c0a4dbb..9f2f4d3 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -64,45 +64,47 @@ def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, S return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) } -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { - try { - sh """#!/bin/bash - # Exit if any command fails - set -xe +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { + withCredentials([usernamePassword(credentialsId: credentials_id, + usernameVariable: 'SUDO_USERNAME', + passwordVariable: 'SUDO_PASSWORD')]) { + def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" + def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" + try { + sh """#!/bin/bash + # Exit if any command fails + set -xe - echo "${password}" | sudo -S echo + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" - sudo chown -v -R root: "${packagedefs.DEBPATH}" + mkdir -v -p "${packageout}" + echo "Building package:" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" - mkdir -v -p "${packageout}" - echo "Building package:" - dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + mkdir -v -p "${uniquepackageout}" + cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" - mkdir -v -p "${uniquepackageout}" - cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" + echo "Showing package:" + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package contents:" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package:" - dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package contents:" - dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package lintian output:" + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - echo "Showing package lintian output:" - lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + """ + } + catch (Exception e) { + echo "Exception occurred" + sh """#!/bin/bash -v + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" - """ - } - catch (Exception e) { - echo "Exception occurred" - sh """#!/bin/bash -v - echo "${password}" | sudo -S echo - - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" - - exit 1 - """ + exit 1 + """ + } } } From 4132c4d470a21475f32a8e0d23a60236f00a4a6d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 22:56:07 +0000 Subject: [PATCH 014/102] Walk a tree of imported meta --- meta/folder.groovy | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 5edde8f..bf14e60 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -930,7 +930,36 @@ class MetaInfo extends GISJsonSerialised { return releasemetaversionuniquecollection } - def walkrelated(ArrayList metacollection, String comment = '', int level = 0) { + def openchildmeta(String gatheredpath, RelatedMetaInfo relatedmetainfo) { + MetaInfo mi = new MetaInfot(script) + + String nextpath + if (gatheredpath) { + nextpath = relatedmetainfo.offsetpath + } else { + nextpath = gatheredpath + '/' + relatedmetainfo.offsetpath + } + def importedmetafile = nextpath + '/' + METAFILENAME + def boolean next + if (script.fileExists(importedmetafile)) { + script.echo("Deserialising meta at file: ${importedmetafile}") + mi.deserialiseJson(importedmetafile) + next = true + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") + next = false + } + + def childmetainfo = [ + next: next + nextpath: nextpath, + childmeta: mi + ] + + return childmetainfo + } + + def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '') { def results = [ 'metacollection' : metacollection, 'comment': comment @@ -938,8 +967,10 @@ class MetaInfo extends GISJsonSerialised { results.metacollection.add(this) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated(results.metacollection, results.comment, level + 1) + def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) + if (childmetainfo.next) { + results = childmetainfo.mi.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) + } } return [ 'metacollection' : results.metacollection, 'comment': results.comment ] } From 71349d97bfcf42292051f505c5c8e26a8d748ce4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 22:58:22 +0000 Subject: [PATCH 015/102] Walk a tree of imported meta --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index bf14e60..a937d74 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -933,7 +933,7 @@ class MetaInfo extends GISJsonSerialised { def openchildmeta(String gatheredpath, RelatedMetaInfo relatedmetainfo) { MetaInfo mi = new MetaInfot(script) - String nextpath + def String nextpath if (gatheredpath) { nextpath = relatedmetainfo.offsetpath } else { @@ -951,9 +951,9 @@ class MetaInfo extends GISJsonSerialised { } def childmetainfo = [ - next: next - nextpath: nextpath, - childmeta: mi + 'next': next + 'nextpath': nextpath, + 'childmeta': mi ] return childmetainfo From a87cb8b649c88183a216c59933879b54c437e426 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 22:59:06 +0000 Subject: [PATCH 016/102] Walk a tree of imported meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a937d74..7d800cc 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -951,7 +951,7 @@ class MetaInfo extends GISJsonSerialised { } def childmetainfo = [ - 'next': next + 'next': next, 'nextpath': nextpath, 'childmeta': mi ] From 0d396e02853735141b646ab4e62d4b9e8f4e2d21 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 22:59:50 +0000 Subject: [PATCH 017/102] Walk a tree of imported meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7d800cc..0473de6 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -931,7 +931,7 @@ class MetaInfo extends GISJsonSerialised { } def openchildmeta(String gatheredpath, RelatedMetaInfo relatedmetainfo) { - MetaInfo mi = new MetaInfot(script) + MetaInfo mi = new MetaInfo(script) def String nextpath if (gatheredpath) { From 3447179c59fa2a38594b907be633c57502b761ef Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 23:06:28 +0000 Subject: [PATCH 018/102] Walk a tree of imported meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0473de6..6e4fd51 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -934,7 +934,7 @@ class MetaInfo extends GISJsonSerialised { MetaInfo mi = new MetaInfo(script) def String nextpath - if (gatheredpath) { + if (gatheredpath == '') { nextpath = relatedmetainfo.offsetpath } else { nextpath = gatheredpath + '/' + relatedmetainfo.offsetpath From 62005036425f2ed68057e73f872fcb4dc6a51e8d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 23:18:28 +0000 Subject: [PATCH 019/102] Walk a tree of imported meta --- meta/folder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6e4fd51..4ffb125 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -931,15 +931,17 @@ class MetaInfo extends GISJsonSerialised { } def openchildmeta(String gatheredpath, RelatedMetaInfo relatedmetainfo) { - MetaInfo mi = new MetaInfo(script) + def MetaInfo mi = new MetaInfo(script) def String nextpath + def importedmetafile if (gatheredpath == '') { nextpath = relatedmetainfo.offsetpath + importedmetafile = nextpath + '/' + metafilepath() } else { nextpath = gatheredpath + '/' + relatedmetainfo.offsetpath + importedmetafile = nextpath + '/' + METAFILENAME } - def importedmetafile = nextpath + '/' + METAFILENAME def boolean next if (script.fileExists(importedmetafile)) { script.echo("Deserialising meta at file: ${importedmetafile}") From 3b446d07bdf79015cb83321d7bb571653c47916c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 23:25:42 +0000 Subject: [PATCH 020/102] Walk a tree of imported meta --- meta/folder.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 4ffb125..a717654 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -942,21 +942,22 @@ class MetaInfo extends GISJsonSerialised { nextpath = gatheredpath + '/' + relatedmetainfo.offsetpath importedmetafile = nextpath + '/' + METAFILENAME } - def boolean next + def boolean imported if (script.fileExists(importedmetafile)) { script.echo("Deserialising meta at file: ${importedmetafile}") mi.deserialiseJson(importedmetafile) - next = true + imported = true } else { script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") - next = false + imported = false } def childmetainfo = [ - 'next': next, + 'imported': imported, 'nextpath': nextpath, 'childmeta': mi ] + script.echo("Child meta results: ${childmetainfo}") return childmetainfo } From 84c313a8b1842b1ec9c9f906a14bf545ddbd7963 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 23:29:26 +0000 Subject: [PATCH 021/102] Walk a tree of imported meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a717654..a4f27c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -971,7 +971,7 @@ class MetaInfo extends GISJsonSerialised { results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) - if (childmetainfo.next) { + if (childmetainfo.imported) { results = childmetainfo.mi.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) } } From 04d17c1b683d4822ab2a6adce9c646c86379d6f9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Dec 2021 23:40:03 +0000 Subject: [PATCH 022/102] Walk a tree of imported meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a4f27c4..38ecdff 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -957,8 +957,6 @@ class MetaInfo extends GISJsonSerialised { 'nextpath': nextpath, 'childmeta': mi ] - script.echo("Child meta results: ${childmetainfo}") - return childmetainfo } @@ -969,8 +967,10 @@ class MetaInfo extends GISJsonSerialised { ] results.metacollection.add(this) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' + script.echo("Partial results: ${results.comment}") for (relatedmetainfo in associatedmeta) { def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) + script.echo("Child meta results: ${childmetainfo}") if (childmetainfo.imported) { results = childmetainfo.mi.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) } From af5f19b4ef0fad336b758d923ccc2d5ec2166950 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 09:11:19 +0000 Subject: [PATCH 023/102] Walk a tree of imported meta --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 38ecdff..c2caaa1 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -961,6 +961,7 @@ class MetaInfo extends GISJsonSerialised { } def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '') { + script.echo("walkrelated: metacollection: ${metacollection}\nComment: ${comment}\nLevel: ${level}\nGathered path: ${gatheredpath}") def results = [ 'metacollection' : metacollection, 'comment': comment From 035977333b644c62082af72c5d1c06c0e7102411 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 09:16:02 +0000 Subject: [PATCH 024/102] Walk a tree of imported meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index c2caaa1..7a6ba4e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -973,7 +973,7 @@ class MetaInfo extends GISJsonSerialised { def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) script.echo("Child meta results: ${childmetainfo}") if (childmetainfo.imported) { - results = childmetainfo.mi.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) + results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) } } return [ 'metacollection' : results.metacollection, 'comment': results.comment ] From 6bf8b5886c912526f7db8f09e13183cd4094afe9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 09:43:38 +0000 Subject: [PATCH 025/102] Walk a tree of imported meta --- meta/folder.groovy | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7a6ba4e..71514fb 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -934,17 +934,14 @@ class MetaInfo extends GISJsonSerialised { def MetaInfo mi = new MetaInfo(script) def String nextpath - def importedmetafile if (gatheredpath == '') { - nextpath = relatedmetainfo.offsetpath - importedmetafile = nextpath + '/' + metafilepath() + nextpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" } else { - nextpath = gatheredpath + '/' + relatedmetainfo.offsetpath - importedmetafile = nextpath + '/' + METAFILENAME + nextpath = "${gatheredpath}/${relatedmetainfo.offsetpath}" } + def importedmetafile = "${nextpath}/${METAFILENAME}" def boolean imported if (script.fileExists(importedmetafile)) { - script.echo("Deserialising meta at file: ${importedmetafile}") mi.deserialiseJson(importedmetafile) imported = true } else { From a64688112a8a5690a2bebc281bc3957e8b0ff5d8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 12:02:01 +0000 Subject: [PATCH 026/102] Remove debug; RelatedMetaInfo does not store meta info anymore --- meta/folder.groovy | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 71514fb..045c866 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -595,28 +595,23 @@ class MetaInfo extends GISJsonSerialised { private String format private String importpath private String offsetpath - private MetaInfo meta - - RelatedMetaInfo(script, importpath = "", offsetpath = "", MetaInfo meta = null) { + + RelatedMetaInfo(script, importpath = "", offsetpath = "") { super(script) this.format = '0.2' this.importpath = importpath this.offsetpath = offsetpath - this.meta = meta } void deserialiseJson(rawmeta) { //format = rawmeta['format'] offsetpath = rawmeta['offsetpath'] - //meta = new MetaInfo(script) - //meta.deserialiseJson(rawmeta['meta']) } LinkedHashMap serialisationdata() { def data = [ 'format' : format, 'offsetpath' : offsetpath, - //'meta' : meta.serialisationdata(), ] return data } @@ -788,7 +783,7 @@ class MetaInfo extends GISJsonSerialised { mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, importedmetafile, adjustedpath, mi) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, importedmetafile, adjustedpath) if (appendtoparentmeta) { associatedmeta.add(rmi) } @@ -958,17 +953,14 @@ class MetaInfo extends GISJsonSerialised { } def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '') { - script.echo("walkrelated: metacollection: ${metacollection}\nComment: ${comment}\nLevel: ${level}\nGathered path: ${gatheredpath}") def results = [ 'metacollection' : metacollection, 'comment': comment ] results.metacollection.add(this) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' - script.echo("Partial results: ${results.comment}") for (relatedmetainfo in associatedmeta) { def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) - script.echo("Child meta results: ${childmetainfo}") if (childmetainfo.imported) { results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) } From 2ba37172e4138a643a9f95975f11f15a37a6d225 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 13:54:14 +0000 Subject: [PATCH 027/102] Handle v0.1 and v0.2 formats simultaneously --- meta/folder.groovy | 91 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 045c866..4f4cb52 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -590,15 +590,52 @@ class MetaInfo extends GISJsonSerialised { private static String METAFOLDERNAME = 'meta' private static String METAFILENAME = 'info.json' private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String format = '0.2' - class RelatedMetaInfo extends GISJsonSerialised { + trait RelatedMetaInfo extends GISJsonSerialised { private String format + + RelatedMetaInfo(script, format) { + super(script) + this.format = format + } + + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + } + + class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { + String offsetpath + MetaInfo meta + + RelatedMetaInfo_v0_1(script, offsetpath = "", MetaInfo meta = null) { + super(script, '0.1') + this.offsetpath = offsetpath + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'meta' : meta.serialisationdata(), + ] + return data + } + } + + class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { private String importpath private String offsetpath - RelatedMetaInfo(script, importpath = "", offsetpath = "") { - super(script) - this.format = '0.2' + RelatedMetaInfo_v0_2(script, importpath = "", offsetpath = "") { + super(script, '0.2') this.importpath = importpath this.offsetpath = offsetpath } @@ -617,7 +654,6 @@ class MetaInfo extends GISJsonSerialised { } } - private String format = '0.1' private ProductInfo productinfo private GitInfo gitinfo private JenkinsBuildInfo jenkinsbuildinfo @@ -705,7 +741,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - if (rawmeta['format'] != '0.1' ) { + if (rawmeta['format'] != '0.1' || rawmeta['format'] != '0.2' ) { script.error("Unsupported meta format found at ${filepath}") return } @@ -727,9 +763,15 @@ class MetaInfo extends GISJsonSerialised { } custominfo = rawmeta['custominfo'] for (rawrelatedmeta in rawmeta['associatedmeta']) { - def rm = new RelatedMetaInfo(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) + if (rawrelatedmeta['format'] == nul || rawrelatedmeta['format'] == '0.1') { + def rm = new RelatedMetaInfo_v0_1(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } else if (rawrelatedmeta['format'] == '0.2') { + def rm = new RelatedMetaInfo_v0_2(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } } } @@ -783,7 +825,7 @@ class MetaInfo extends GISJsonSerialised { mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, importedmetafile, adjustedpath) + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath) if (appendtoparentmeta) { associatedmeta.add(rmi) } @@ -952,6 +994,24 @@ class MetaInfo extends GISJsonSerialised { return childmetainfo } + def walkrelated_v0_1(ArrayList metacollection, String comment = '', int level = 0) { + def results = [ + 'metacollection' : metacollection, + 'comment': comment + ] + results.metacollection.add(this) + results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1) + } else { + script.error("There is non v0.1 format related meta info under v0.1 related meta info") + } + } + return [ 'metacollection' : results.metacollection, 'comment': results.comment ] + } + def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '') { def results = [ 'metacollection' : metacollection, @@ -960,9 +1020,14 @@ class MetaInfo extends GISJsonSerialised { results.metacollection.add(this) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { - def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) - if (childmetainfo.imported) { - results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) + if (relatedmetainfo.format == '0.2') { + def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) + if (childmetainfo.imported) { + results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) + } + } else if (relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1) } } return [ 'metacollection' : results.metacollection, 'comment': results.comment ] From f31914d43b355effb283763335302a9628688ca6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 14:10:28 +0000 Subject: [PATCH 028/102] Make sure all jobs are using the new archiver --- copy/esw.groovy | 2 +- copy/filtered.groovy | 2 +- copy/logic.groovy | 2 +- staticanalysis/cpd.groovy | 2 +- staticanalysis/cppcheck.groovy | 2 +- test/testethernetproduct.groovy | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/copy/esw.groovy b/copy/esw.groovy index 6ccb53e..5d9af25 100644 --- a/copy/esw.groovy +++ b/copy/esw.groovy @@ -42,7 +42,7 @@ pipeline { mv -v "${ESW}"/* . """ artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(artifactlist) + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/copy/filtered.groovy b/copy/filtered.groovy index 251de93..555ce8d 100644 --- a/copy/filtered.groovy +++ b/copy/filtered.groovy @@ -49,7 +49,7 @@ pipeline { script { def artifactlist = meta.generateartifacts(["filtered/**"]) artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(artifactlist) + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/copy/logic.groovy b/copy/logic.groovy index 7b20935..0a28a22 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -44,7 +44,7 @@ pipeline { mv -v "logic"/* . """ artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(artifactlist) + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/staticanalysis/cpd.groovy b/staticanalysis/cpd.groovy index f9ddfcb..c04649c 100644 --- a/staticanalysis/cpd.groovy +++ b/staticanalysis/cpd.groovy @@ -130,7 +130,7 @@ pipeline { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(artifactlist) + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index 696f11d..d959770 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -92,7 +92,7 @@ pipeline { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(artifactlist) + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/test/testethernetproduct.groovy b/test/testethernetproduct.groovy index f45cf19..5e2ce53 100644 --- a/test/testethernetproduct.groovy +++ b/test/testethernetproduct.groovy @@ -193,7 +193,7 @@ pipeline { script { def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(artifactlist) + artifactarchiver.archive(meta, artifactlist) } } } From 9e0fe09f0efc16bfb4be0a8e70282aa897f41757 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 15:15:58 +0000 Subject: [PATCH 029/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 4f4cb52..cd5fe35 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -592,7 +592,13 @@ class MetaInfo extends GISJsonSerialised { private static String METAARTIFACTS = "${METAFOLDERNAME}/**" private static String format = '0.2' - trait RelatedMetaInfo extends GISJsonSerialised { + interface IRelatedMetaInfo { + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + abstract String format() + } + + class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format RelatedMetaInfo(script, format) { @@ -602,6 +608,10 @@ class MetaInfo extends GISJsonSerialised { abstract void deserialiseJson(rawmeta) abstract LinkedHashMap serialisationdata() + + String format() { + return format + } } class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { From 0406350f78410b11eb7041b2406f55e1c473cdec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 15:19:35 +0000 Subject: [PATCH 030/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cd5fe35..4262a1c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -598,7 +598,7 @@ class MetaInfo extends GISJsonSerialised { abstract String format() } - class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { + abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format RelatedMetaInfo(script, format) { From 6502cf749245ed689c1d3366a205e2d3bdbe4d3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 15:25:32 +0000 Subject: [PATCH 031/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 4262a1c..1b409ff 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -898,7 +898,7 @@ class MetaInfo extends GISJsonSerialised { private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { if (maxdepth > 0) { if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_1(script, pathsofar, this) releasemetacollection.add(rmi) } if (pathsofar) { From e57708a515daa4840eb46156102fc71f8aa7cb88 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 15:44:01 +0000 Subject: [PATCH 032/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1b409ff..143d29b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -751,8 +751,9 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + script.echo("Format: ${rawmeta['format']}") if (rawmeta['format'] != '0.1' || rawmeta['format'] != '0.2' ) { - script.error("Unsupported meta format found at ${filepath}") + script.error("Unsupported meta format found") return } productinfo.deserialiseJson(rawmeta['productinfo']) From e9157c3b63e2cf10ca99e12d78602c1207dce3c1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 15:48:35 +0000 Subject: [PATCH 033/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 143d29b..2651e70 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -751,8 +751,8 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - script.echo("Format: ${rawmeta['format']}") - if (rawmeta['format'] != '0.1' || rawmeta['format'] != '0.2' ) { + script.echo("Format: ${rawmeta['format']} ${rawmeta['format'].getClass()}") + if (rawmeta['format'].toString() != '0.1' || rawmeta['format'].toString() != '0.2' ) { script.error("Unsupported meta format found") return } From 149e1fd8f4e2fdb41535f9d3203dc6102b9c3184 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 16:01:30 +0000 Subject: [PATCH 034/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2651e70..ad88d7c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -751,8 +751,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - script.echo("Format: ${rawmeta['format']} ${rawmeta['format'].getClass()}") - if (rawmeta['format'].toString() != '0.1' || rawmeta['format'].toString() != '0.2' ) { + if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { script.error("Unsupported meta format found") return } From 523407f68fa9dcba0f2813f62471eb2c79f51ba2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 16:04:59 +0000 Subject: [PATCH 035/102] Make sure all jobs are using the new archiver --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ad88d7c..dd3a1b7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -773,7 +773,7 @@ class MetaInfo extends GISJsonSerialised { } custominfo = rawmeta['custominfo'] for (rawrelatedmeta in rawmeta['associatedmeta']) { - if (rawrelatedmeta['format'] == nul || rawrelatedmeta['format'] == '0.1') { + if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { def rm = new RelatedMetaInfo_v0_1(script) rm.deserialiseJson(rawrelatedmeta) associatedmeta.add(rm) From c985dcb55234f492d5b945d85c791656815745a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 17:20:28 +0000 Subject: [PATCH 036/102] Use non-deprecated methods for importing --- artifacts/metaexport.groovy | 7 ++++++- copy/esw.groovy | 5 ++++- copy/filtered.groovy | 4 +++- differences/metahierarchy.groovy | 14 ++++++++++++-- publish/package.groovy | 7 ++++++- 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 4d93cc5..8e5e56e 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -11,7 +11,12 @@ def combineSubjobArtifacts(meta, importer, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(subJobName, subJobNumber) + importoptions.setincrementversionnumber(incrementversionnumber) + importer.artifactimport(meta, importoptions, destFolder) + artifactFolders.add("${destFolder}/**") } } diff --git a/copy/esw.groovy b/copy/esw.groovy index 5d9af25..b9eb737 100644 --- a/copy/esw.groovy +++ b/copy/esw.groovy @@ -28,7 +28,10 @@ pipeline { stage('Copying embedded software artifacts') { steps { script { - importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST) + def eswimportoptions = importer.getNewImportOptions() + eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) + eswimportoptions.setincrementversionnumber(true) + importer.artifactimport(meta, eswimportoptions, ESWDEST) } } } diff --git a/copy/filtered.groovy b/copy/filtered.groovy index 555ce8d..3ad24cd 100644 --- a/copy/filtered.groovy +++ b/copy/filtered.groovy @@ -27,7 +27,9 @@ pipeline { stage('Import artifacts') { steps { script { - importer.artifactimport(meta, ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION, IMPORT) + def importoptions = importer.getNewImportOptions() + importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) + importer.artifactimport(meta, importoptions, IMPORT) } } } diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 60d9835..01c1890 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -37,8 +37,18 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') - importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') + + def earlyimportoptions = importer.getNewImportOptions() + earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) + earlyimportoptions.setincrementversionnumber(false) + earlyimportoptions.setfilter('meta/**') + importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) + + def laterimportoptions = importer.getNewImportOptions() + laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) + laterimportoptions.setincrementversionnumber(false) + laterimportoptions.setfilter('meta/**') + importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) } } } diff --git a/publish/package.groovy b/publish/package.groovy index 958a614..68c41f3 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -48,7 +48,12 @@ pipeline { steps { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) + + def packageimportoptions = importer.getNewImportOptions() + packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) + packageimportoptions.setincrementversionnumber(true) + packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) + PACKAGE_FOLDER = "${PACKAGEDEST}/package" UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" } From 9b2752021245f9a3cfdfbc531eab2176d891240d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 Dec 2021 21:59:26 +0000 Subject: [PATCH 037/102] Make all walkers use a walker that understand v0.1 and v0.2 formats --- meta/folder.groovy | 137 ++++++++++++++++++--------------------------- 1 file changed, 54 insertions(+), 83 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index dd3a1b7..adf124a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -893,31 +893,6 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } - - // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java - private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { - if (maxdepth > 0) { - if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo_v0_1(script, pathsofar, this) - releasemetacollection.add(rmi) - } - if (pathsofar) { - pathsofar += "/" - } - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + relatedmetainfo.offsetpath - releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) - } - } - return releasemetacollection - } - - ArrayList releasewalker(int maxdepth, String startpath = "") { - ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - return releasemetacollection - } String componentidentifier() { return productinfo.productname + "_" + productinfo.subcomponentname @@ -939,44 +914,6 @@ class MetaInfo extends GISJsonSerialised { return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber } - Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { - if (releasemetacollection.containsKey(uniquestring)) { - def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { - releasemetacollection.put(uniquestring, relatedmetainfo) - } - } else { - releasemetacollection.put(uniquestring, relatedmetainfo) - } - return releasemetacollection - } - - Map releasewalkerunique(int maxdepth, String startpath = "") { - def ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - - def Map releasemetauniquecollection = new LinkedHashMap() - for (relatedmetainfo in releasemetacollection) { - def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) - } - - return releasemetauniquecollection - } - - Map releasewalkerversionunique(int maxdepth, String startpath = "") { - ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - - def Map releasemetaversionuniquecollection = new LinkedHashMap() - for (relatedmetainfo in releasemetacollection) { - def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) - } - - return releasemetaversionuniquecollection - } - def openchildmeta(String gatheredpath, RelatedMetaInfo relatedmetainfo) { def MetaInfo mi = new MetaInfo(script) @@ -1004,40 +941,44 @@ class MetaInfo extends GISJsonSerialised { return childmetainfo } - def walkrelated_v0_1(ArrayList metacollection, String comment = '', int level = 0) { + def walkrelated_v0_1(ArrayList metacollection, String comment = '', int level = 0, boolean limiteddepth = false, int maxdepth = 0) { def results = [ 'metacollection' : metacollection, 'comment': comment ] - results.metacollection.add(this) - results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { - def MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1) - } else { - script.error("There is non v0.1 format related meta info under v0.1 related meta info") + if (!limiteddepth || maxdepth > 0) { + results.metacollection.add(this) + results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, limiteddepth, maxdepth - 1) + } else { + script.error("There is non v0.1 format related meta info under v0.1 related meta info") + } } } return [ 'metacollection' : results.metacollection, 'comment': results.comment ] } - def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '') { + def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '', boolean limiteddepth = false, int maxdepth = 0) { def results = [ 'metacollection' : metacollection, 'comment': comment ] - results.metacollection.add(this) - results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - if (relatedmetainfo.format == '0.2') { - def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) - if (childmetainfo.imported) { - results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath) + if (!limiteddepth || maxdepth > 0) { + results.metacollection.add(this) + results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (relatedmetainfo.format == '0.2') { + def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) + if (childmetainfo.imported) { + results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath, limiteddepth, maxdepth - 1) + } + } else if (relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, limiteddepth, maxdepth - 1) } - } else if (relatedmetainfo.format == '0.1') { - def MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1) } } return [ 'metacollection' : results.metacollection, 'comment': results.comment ] @@ -1048,6 +989,36 @@ class MetaInfo extends GISJsonSerialised { return walkrelated(metacollection) } + ArrayList releasewalker(int maxdepth, String startpath = '') { + def ArrayList metacollection = new ArrayList() + def results = walkrelated(metacollection, '', 0, startpath, true, maxdepth) + return results.metacollection + } + + Map releasewalkerversionunique(int maxdepth, String startpath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath) + + def Map versionuniquecollection = new LinkedHashMap() + for (metainfo in metacollection) { + def String uniquestring = metainfo.componentversionuniqueidentifier() + versionuniquecollection.put(uniquestring, metainfo) + } + + return versionuniquecollection + } + + Map releasewalkerunique(int maxdepth, String startpath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath) + + def Map uniquecollection = new LinkedHashMap() + for (metainfo in metacollection) { + def String uniquestring = metainfo.componentidentifier() + uniquecollection.put(uniquestring, metainfo) + } + + return uniquecollection + } + MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } From 1ee2be62fff8ed870aee9433252fa30800d2bd40 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:11:39 +0000 Subject: [PATCH 038/102] Use importer to get build artifacts to release --- release/release.groovy | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/release/release.groovy b/release/release.groovy index 42357fc..5c9925e 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -23,18 +23,15 @@ pipeline { stage('Get build artifact meta') { steps { script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" - copyArtifacts( - filter: 'meta/**', - projectName: BUILD_JOB, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR - ], - target: BUILD_ARTIFACTS - ) - def mfs = autobuilderscript.getmetafolderscript() - buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) + importoptions.setincrementversionnumber(false) + importoptions.setfilter('meta/**') + buildmeta = importer.artifactimport(meta, controlimportoptions, BUILD_ARTIFACTS) } } } From 7d8e910d202e2ee029d7d5dc91cdf48afb10575b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:12:47 +0000 Subject: [PATCH 039/102] Use importer to get build artifacts to release --- release/release.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/release.groovy b/release/release.groovy index 5c9925e..918c386 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -31,7 +31,7 @@ pipeline { importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) importoptions.setincrementversionnumber(false) importoptions.setfilter('meta/**') - buildmeta = importer.artifactimport(meta, controlimportoptions, BUILD_ARTIFACTS) + buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) } } } From 96621af758c19411cdd68230ac773f140b76c511 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:27:21 +0000 Subject: [PATCH 040/102] Use importer to get build artifacts to release --- release/release.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/release.groovy b/release/release.groovy index 918c386..17f5185 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -25,7 +25,7 @@ pipeline { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + BUILD_ARTIFACTS = "${WORKSPACE}" def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) From aff9e3e106c195703501e06f633fe178503cb7aa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:31:46 +0000 Subject: [PATCH 041/102] Use importer to get build artifacts to release --- release/release.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/release.groovy b/release/release.groovy index 17f5185..8f7d129 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -25,7 +25,7 @@ pipeline { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - BUILD_ARTIFACTS = "${WORKSPACE}" + BUILD_ARTIFACTS = '.' def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) From f3a0f9fcc95a886543e22e0611c6c188484b423f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:35:17 +0000 Subject: [PATCH 042/102] Use importer to get build artifacts to release --- release/release.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/release.groovy b/release/release.groovy index 8f7d129..daf42aa 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -31,6 +31,7 @@ pipeline { importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) importoptions.setincrementversionnumber(false) importoptions.setfilter('meta/**') + importoptions.clean(false) buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) } } From fc34f642388f769ae727dd3c73bfe4bfd6e5009f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:37:12 +0000 Subject: [PATCH 043/102] Use importer to get build artifacts to release --- release/release.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release/release.groovy b/release/release.groovy index daf42aa..2c168b9 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -25,13 +25,12 @@ pipeline { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - BUILD_ARTIFACTS = '.' + BUILD_ARTIFACTS = 'build' def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) importoptions.setincrementversionnumber(false) importoptions.setfilter('meta/**') - importoptions.clean(false) buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) } } From 135fddb356051307cf93e02d2e90bd47a0be84df Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:41:45 +0000 Subject: [PATCH 044/102] Use importer to get build artifacts to release --- release/release.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/release.groovy b/release/release.groovy index 2c168b9..20f520c 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -31,6 +31,7 @@ pipeline { importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) importoptions.setincrementversionnumber(false) importoptions.setfilter('meta/**') + importoptions.setappendtoparentmeta(false) buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) } } From e35826fb24014d77f52fae247010b775c190da5e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:45:18 +0000 Subject: [PATCH 045/102] Use importer to get build artifacts to release --- release/do.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index e0b9471..3430342 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -396,13 +396,16 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri repoandtagsappend("\n") } -def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { +def releasehierarchy(buildfolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { def int md = maxdepth.toInteger() resetfolders() preconditions() // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo = buildmeta.releasewalkerunique(md) + def componentReleaseInfo + dir(buildfolder) { + componentReleaseInfo = buildmeta.releasewalkerunique(md) + } echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { From b64672cc94017ad76223e26e9eaa9b801e1ab2fc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:47:28 +0000 Subject: [PATCH 046/102] Use importer to get build artifacts to release --- release/release.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/release.groovy b/release/release.groovy index 20f520c..2324b60 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -42,7 +42,7 @@ pipeline { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" - (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' From f41768212cc3352a3e01e424d118253a5b31f870 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 14:56:32 +0000 Subject: [PATCH 047/102] Use importer to get build artifacts to release --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index adf124a..88e05e5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -991,7 +991,7 @@ class MetaInfo extends GISJsonSerialised { ArrayList releasewalker(int maxdepth, String startpath = '') { def ArrayList metacollection = new ArrayList() - def results = walkrelated(metacollection, '', 0, startpath, true, maxdepth) + def results = walkrelated(metacollection, 'meta', 0, startpath, true, maxdepth) return results.metacollection } From 82dc66da2f754714e99b91e9e29845ff7dcfb37b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 21:34:55 +0000 Subject: [PATCH 048/102] Use importer to get build artifacts to release --- meta/folder.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 88e05e5..499535a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -942,6 +942,7 @@ class MetaInfo extends GISJsonSerialised { } def walkrelated_v0_1(ArrayList metacollection, String comment = '', int level = 0, boolean limiteddepth = false, int maxdepth = 0) { + script.echo("walkrelated_v0_1 Iteration:\nLevel: ${level}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}") def results = [ 'metacollection' : metacollection, 'comment': comment @@ -962,6 +963,7 @@ class MetaInfo extends GISJsonSerialised { } def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '', boolean limiteddepth = false, int maxdepth = 0) { + script.echo("walkrelated Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}") def results = [ 'metacollection' : metacollection, 'comment': comment From 5b350028ea1a0391199b6f014a389dc31e69cd04 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 22:02:04 +0000 Subject: [PATCH 049/102] Try to walk through build meta items --- release/do.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 3430342..563a49f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -402,10 +402,7 @@ def releasehierarchy(buildfolder, buildmeta, String maxdepth, String otherreleas preconditions() // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo - dir(buildfolder) { - componentReleaseInfo = buildmeta.releasewalkerunique(md) - } + def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { From 99aa54ae8e6908083bbf02a1f49e62ddb12f956b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 22:13:33 +0000 Subject: [PATCH 050/102] Try to walk through build meta items --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 499535a..3f60bfa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -998,7 +998,7 @@ class MetaInfo extends GISJsonSerialised { } Map releasewalkerversionunique(int maxdepth, String startpath = '') { - def ArrayList metacollection = releasewalker(maxdepth, startpath) + def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}") def Map versionuniquecollection = new LinkedHashMap() for (metainfo in metacollection) { @@ -1010,7 +1010,7 @@ class MetaInfo extends GISJsonSerialised { } Map releasewalkerunique(int maxdepth, String startpath = '') { - def ArrayList metacollection = releasewalker(maxdepth, startpath) + def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}") def Map uniquecollection = new LinkedHashMap() for (metainfo in metacollection) { From b973208999feeea379038cc13a2297495fc02c5b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 Dec 2021 22:20:07 +0000 Subject: [PATCH 051/102] Try to walk through build meta items --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 563a49f..d131173 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -411,7 +411,7 @@ def releasehierarchy(buildfolder, buildmeta, String maxdepth, String otherreleas echo versioncomponent // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder) echo "Version unique components:" def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { From ece26703656703826e0ab36307710b30d0181031 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 Dec 2021 15:39:32 +0000 Subject: [PATCH 052/102] Make the walkers carry more meta data --- meta/folder.groovy | 111 +++++++++++++++++++++++++++++++-------------- 1 file changed, 76 insertions(+), 35 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 3f60bfa..fb10cbd 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -914,16 +914,32 @@ class MetaInfo extends GISJsonSerialised { return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber } - def openchildmeta(String gatheredpath, RelatedMetaInfo relatedmetainfo) { + class CollectedMeta { + private MetaInfo metainfo + private String accumulatedpath + + CollectedMeta(MetaInfo metainfo, String accumulatedpath = '') { + this.metainfo = metainfo + this.accumulatedpath = accumulatedpath + } + } + + def openchildmeta(String gatheredpath, String gatheredmetapath, RelatedMetaInfo relatedmetainfo) { def MetaInfo mi = new MetaInfo(script) - def String nextpath + def String nextgatheredpath if (gatheredpath == '') { - nextpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + nextgatheredpath = "${relatedmetainfo.offsetpath}" } else { - nextpath = "${gatheredpath}/${relatedmetainfo.offsetpath}" + nextgatheredpath = "${gatheredpath}/${relatedmetainfo.offsetpath}" } - def importedmetafile = "${nextpath}/${METAFILENAME}" + def String nextgatheredmetapath + if (gatheredmetapath == '') { + nextgatheredmetapath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + } else { + nextgatheredmetapath = "${gatheredmetapath}/${relatedmetainfo.offsetpath}" + } + def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" def boolean imported if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) @@ -935,25 +951,38 @@ class MetaInfo extends GISJsonSerialised { def childmetainfo = [ 'imported': imported, - 'nextpath': nextpath, + 'nextgatheredpath': nextgatheredpath, + 'nextgatheredmetapath': nextgatheredmetapath, 'childmeta': mi ] return childmetainfo } - def walkrelated_v0_1(ArrayList metacollection, String comment = '', int level = 0, boolean limiteddepth = false, int maxdepth = 0) { - script.echo("walkrelated_v0_1 Iteration:\nLevel: ${level}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}") + def walkrelated_v0_1(ArrayList metacollection, + String comment = '', + int level = 0, + String gatheredpath = '', + String gatheredmetapath = '', + boolean limiteddepth = false, + int maxdepth = 0) { + script.echo("walkrelated_v0_1 Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}") def results = [ 'metacollection' : metacollection, 'comment': comment ] if (!limiteddepth || maxdepth > 0) { - results.metacollection.add(this) + results.metacollection.add(new CollectedMeta(this, gatheredpath)) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { def MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, limiteddepth, maxdepth - 1) + results = othermeta.walkrelated_v0_1(results.metacollection, + results.comment, + level + 1, + "${gatheredpath}/${relatedmetainfo.offsetpath}", + "${gatheredmetapath}/${relatedmetainfo.offsetpath}", + limiteddepth, + maxdepth - 1) } else { script.error("There is non v0.1 format related meta info under v0.1 related meta info") } @@ -962,24 +991,36 @@ class MetaInfo extends GISJsonSerialised { return [ 'metacollection' : results.metacollection, 'comment': results.comment ] } - def walkrelated(ArrayList metacollection, String comment = '', int level = 0, String gatheredpath = '', boolean limiteddepth = false, int maxdepth = 0) { - script.echo("walkrelated Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}") + def walkrelated(ArrayList metacollection, + String comment = '', + int level = 0, + String gatheredpath = '', + String gatheredmetapath = '', + boolean limiteddepth = false, + int maxdepth = 0) { + script.echo("walkrelated Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}") def results = [ 'metacollection' : metacollection, 'comment': comment ] if (!limiteddepth || maxdepth > 0) { - results.metacollection.add(this) + results.metacollection.add(new CollectedMeta(this, gatheredpath)) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { if (relatedmetainfo.format == '0.2') { - def childmetainfo = openchildmeta(gatheredpath, relatedmetainfo) + def childmetainfo = openchildmeta(gatheredpath, gatheredmetapath, relatedmetainfo) if (childmetainfo.imported) { - results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, childmetainfo.nextpath, limiteddepth, maxdepth - 1) + results = childmetainfo.childmeta.walkrelated(results.metacollection, + results.comment, + level + 1, + childmetainfo.nextgatheredpath + childmetainfo.nextgatheredmetapath, + limiteddepth, + maxdepth - 1) } } else if (relatedmetainfo.format == '0.1') { def MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, limiteddepth, maxdepth - 1) + results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, gatheredpath, gatheredmetapath, limiteddepth, maxdepth - 1) } } } @@ -987,23 +1028,23 @@ class MetaInfo extends GISJsonSerialised { } def allrelatedmeta() { - ArrayList metacollection = new ArrayList() + ArrayList metacollection = new ArrayList() return walkrelated(metacollection) } - ArrayList releasewalker(int maxdepth, String startpath = '') { - def ArrayList metacollection = new ArrayList() - def results = walkrelated(metacollection, 'meta', 0, startpath, true, maxdepth) + ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = new ArrayList() + def results = walkrelated(metacollection, 'meta', 0, startpath, startmetapath, true, maxdepth) return results.metacollection } - Map releasewalkerversionunique(int maxdepth, String startpath = '') { - def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}") + Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}", startmetapath) - def Map versionuniquecollection = new LinkedHashMap() - for (metainfo in metacollection) { - def String uniquestring = metainfo.componentversionuniqueidentifier() - versionuniquecollection.put(uniquestring, metainfo) + def Map versionuniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() + versionuniquecollection.put(uniquestring, metaitem) } return versionuniquecollection @@ -1013,9 +1054,9 @@ class MetaInfo extends GISJsonSerialised { def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}") def Map uniquecollection = new LinkedHashMap() - for (metainfo in metacollection) { - def String uniquestring = metainfo.componentidentifier() - uniquecollection.put(uniquestring, metainfo) + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentidentifier() + uniquecollection.put(uniquestring, metaitem) } return uniquecollection @@ -1027,15 +1068,15 @@ class MetaInfo extends GISJsonSerialised { def findAssociatedMeta(String productname, String subcomponentname) { def results = [ - 'matchedmeta' : getNewMetaObject(this), + 'matchedmetaitem' : new CollectedMeta(this), 'matchfound' : false ] def walkresults = allrelatedmeta() - for (dependentmeta in walkresults.metacollection) { - if (dependentmeta.productinfo.productname == productname && - dependentmeta.productinfo.subcomponentname == subcomponentname) { - script.echo("Found matching component: ${dependentmeta.globalreleasetag()}") - results.matchedmeta = dependentmeta + for (dependentmetaitem in walkresults.metacollection) { + if (dependentmetaitem.metainfo.productinfo.productname == productname && + dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { + script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") + results.matchedmetaitem = dependentmetaitem results.matchfound = true break } From 053b39b3df1bbf2f344cdc8ba7701f7fe94f038f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 11:23:05 +0000 Subject: [PATCH 053/102] Make the walkers carry more meta data --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index fb10cbd..0fd0d8e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1013,7 +1013,7 @@ class MetaInfo extends GISJsonSerialised { results = childmetainfo.childmeta.walkrelated(results.metacollection, results.comment, level + 1, - childmetainfo.nextgatheredpath + childmetainfo.nextgatheredpath, childmetainfo.nextgatheredmetapath, limiteddepth, maxdepth - 1) From 93ad1bea33bd4304e831232dca66af2ee683b69a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 11:50:07 +0000 Subject: [PATCH 054/102] Make the walkers carry more meta data --- meta/folder.groovy | 8 ++++---- release/do.groovy | 6 +++--- release/release.groovy | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0fd0d8e..1c6441b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1039,7 +1039,7 @@ class MetaInfo extends GISJsonSerialised { } Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}", startmetapath) + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") def Map versionuniquecollection = new LinkedHashMap() for (metaitem in metacollection) { @@ -1050,10 +1050,10 @@ class MetaInfo extends GISJsonSerialised { return versionuniquecollection } - Map releasewalkerunique(int maxdepth, String startpath = '') { - def ArrayList metacollection = releasewalker(maxdepth, "${startpath}/${METAFOLDERNAME}") + Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - def Map uniquecollection = new LinkedHashMap() + def Map uniquecollection = new LinkedHashMap() for (metaitem in metacollection) { def String uniquestring = metaitem.metainfo.componentidentifier() uniquecollection.put(uniquestring, metaitem) diff --git a/release/do.groovy b/release/do.groovy index d131173..133d425 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -396,13 +396,13 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri repoandtagsappend("\n") } -def releasehierarchy(buildfolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { +def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { def int md = maxdepth.toInteger() resetfolders() preconditions() // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder) + def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { @@ -411,7 +411,7 @@ def releasehierarchy(buildfolder, buildmeta, String maxdepth, String otherreleas echo versioncomponent // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder) + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder metafolder) echo "Version unique components:" def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { diff --git a/release/release.groovy b/release/release.groovy index 2324b60..82af511 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -42,7 +42,7 @@ pipeline { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" - (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' From 61e2834a1b7c11a77eb0b7675579cb7e044ce073 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 11:58:52 +0000 Subject: [PATCH 055/102] Make the walkers carry more meta data --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 133d425..2f175ea 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -411,7 +411,7 @@ def releasehierarchy(String buildfolder, String metafolder, buildmeta, String ma echo versioncomponent // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder metafolder) + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) echo "Version unique components:" def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { From 4b72bba43e0082fc7beca9399122248e9287771c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 12:29:11 +0000 Subject: [PATCH 056/102] Make the walkers carry more meta data --- meta/folder.groovy | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1c6441b..eef608d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -917,10 +917,12 @@ class MetaInfo extends GISJsonSerialised { class CollectedMeta { private MetaInfo metainfo private String accumulatedpath + private String accumulatedmetapath - CollectedMeta(MetaInfo metainfo, String accumulatedpath = '') { + CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { this.metainfo = metainfo this.accumulatedpath = accumulatedpath + this.accumulatedmetapath = accumulatedmetapath } } @@ -971,7 +973,7 @@ class MetaInfo extends GISJsonSerialised { 'comment': comment ] if (!limiteddepth || maxdepth > 0) { - results.metacollection.add(new CollectedMeta(this, gatheredpath)) + results.metacollection.add(new CollectedMeta(this, gatheredpath, gatheredmetapath)) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { @@ -1004,7 +1006,7 @@ class MetaInfo extends GISJsonSerialised { 'comment': comment ] if (!limiteddepth || maxdepth > 0) { - results.metacollection.add(new CollectedMeta(this, gatheredpath)) + results.metacollection.add(new CollectedMeta(this, gatheredpath, gatheredmetapath)) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { if (relatedmetainfo.format == '0.2') { @@ -1038,13 +1040,25 @@ class MetaInfo extends GISJsonSerialised { return results.metacollection } + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.accumulatedpath.size() > metaitem.accumulatedpath.size()) { + releasemetacollection.put(uniquestring, metaitem) + } + } else { + releasemetacollection.put(uniquestring, metaitem) + } + return releasemetacollection + } + Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") def Map versionuniquecollection = new LinkedHashMap() for (metaitem in metacollection) { def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() - versionuniquecollection.put(uniquestring, metaitem) + versionuniquecollection = putwithshortestoffsetpath(uniquestring, metaitem) } return versionuniquecollection @@ -1056,7 +1070,7 @@ class MetaInfo extends GISJsonSerialised { def Map uniquecollection = new LinkedHashMap() for (metaitem in metacollection) { def String uniquestring = metaitem.metainfo.componentidentifier() - uniquecollection.put(uniquestring, metaitem) + versionuniquecollection = putwithshortestoffsetpath(uniquestring, metaitem) } return uniquecollection From dcd5fe6382f62202e124b296edb500c5d3b1e3ba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 12:31:24 +0000 Subject: [PATCH 057/102] Make the walkers carry more meta data --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index eef608d..9934287 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1058,7 +1058,7 @@ class MetaInfo extends GISJsonSerialised { def Map versionuniquecollection = new LinkedHashMap() for (metaitem in metacollection) { def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() - versionuniquecollection = putwithshortestoffsetpath(uniquestring, metaitem) + versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) } return versionuniquecollection @@ -1070,7 +1070,7 @@ class MetaInfo extends GISJsonSerialised { def Map uniquecollection = new LinkedHashMap() for (metaitem in metacollection) { def String uniquestring = metaitem.metainfo.componentidentifier() - versionuniquecollection = putwithshortestoffsetpath(uniquestring, metaitem) + versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) } return uniquecollection From 55a087b237e35f3a0363bc0db933162ed88b34ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 12:32:48 +0000 Subject: [PATCH 058/102] Make the walkers carry more meta data --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9934287..fb27e64 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1070,7 +1070,7 @@ class MetaInfo extends GISJsonSerialised { def Map uniquecollection = new LinkedHashMap() for (metaitem in metacollection) { def String uniquestring = metaitem.metainfo.componentidentifier() - versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) + uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) } return uniquecollection From a342a0f482410c9eb50f6729267e14baf9c60066 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 12:40:42 +0000 Subject: [PATCH 059/102] Alter release process to handle new meta walker info --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2f175ea..8a2ea61 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -112,10 +112,10 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") def boolean result = true - for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { + for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { def filterpath = artifactfilter - if (buildinfo.offsetpath) { - filterpath = buildinfo.offsetpath + '/' + artifactfilter + if (buildinfo.accumulatedpath) { + filterpath = buildinfo.accumulatedpath + '/' + artifactfilter } // Copy relevant artifacts to temp folder echo "Copying artifacts: ${filterpath} to ${tempfolder}" @@ -136,7 +136,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } if (result) { // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath dir(originalrootdir) { sh("mv * ${repofolder}") } @@ -375,7 +375,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { - def buildmeta = buildinfo.meta + def buildmeta = buildinfo.metainfo def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" @@ -427,7 +427,7 @@ def releasehierarchy(String buildfolder, String metafolder, buildmeta, String ma for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - if (!buildinfo.jenkinsbuildinfo.isFullBuild()) { + if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" } } From 206e0f30a1849f87fe8b302e224a06dd25cc4e52 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 13:10:08 +0000 Subject: [PATCH 060/102] Alter release process to handle new meta walker info --- release/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 8a2ea61..a521427 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -118,14 +118,14 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde filterpath = buildinfo.accumulatedpath + '/' + artifactfilter } // Copy relevant artifacts to temp folder - echo "Copying artifacts: ${filterpath} to ${tempfolder}" + echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" try { copyArtifacts( filter: filterpath, projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" ], target: tempfolder ) @@ -138,7 +138,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath dir(originalrootdir) { - sh("mv * ${repofolder}") + sh("mv -v * ${repofolder}") } } summaryappend("${artifactfilter},") From 0b057d0a0e748898aa869ade3f03d3c762c3e349 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 13:26:48 +0000 Subject: [PATCH 061/102] Alter release process to handle new meta walker info --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index a521427..ca7b84e 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -134,6 +134,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde result = false summaryappend("Does not exist:") } + error("Debugging") if (result) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath From 2fe1414c19ccdbe105575df964dfc62b71d256ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 13:30:57 +0000 Subject: [PATCH 062/102] Alter release process to handle new meta walker info --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index ca7b84e..d5b43ed 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -134,7 +134,6 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde result = false summaryappend("Does not exist:") } - error("Debugging") if (result) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath @@ -143,6 +142,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } } summaryappend("${artifactfilter},") + error("Debugging") } return result } From af1c49958a82af57995d8e77b9d0723306ffad1a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 13:34:51 +0000 Subject: [PATCH 063/102] Alter release process to handle new meta walker info --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index d5b43ed..8c25fc2 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -142,8 +142,8 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } } summaryappend("${artifactfilter},") - error("Debugging") } + error("Debugging") return result } From 2f8ac675867242232ca2003c8866b3bf7c19743f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 13:41:50 +0000 Subject: [PATCH 064/102] Alter release process to handle new meta walker info --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 8c25fc2..e7d936d 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -143,7 +143,6 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } summaryappend("${artifactfilter},") } - error("Debugging") return result } @@ -359,6 +358,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source releasegitpush(dryrunpush) } } + error("Debugging") dir(sourcerepo) { def String srcbuildstamptag = buildmeta.globalreleasetag() def String buildsha = buildmeta.gitinfo.gitrevision From 8ccca6aa15ba6375c808f0c462a735a44162fac8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 13:57:54 +0000 Subject: [PATCH 065/102] Alter release process to handle new meta walker info --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index e7d936d..a521427 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -358,7 +358,6 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source releasegitpush(dryrunpush) } } - error("Debugging") dir(sourcerepo) { def String srcbuildstamptag = buildmeta.globalreleasetag() def String buildsha = buildmeta.gitinfo.gitrevision From 236aa6fe68c8f94dbc235ff95da4b436f3593b5a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 14:11:41 +0000 Subject: [PATCH 066/102] Alter release process to handle new meta walker info --- release/do.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a521427..87eb9ae 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -412,13 +412,18 @@ def releasehierarchy(String buildfolder, String metafolder, buildmeta, String ma // Walk the meta information to a max depth, looking for components to release def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) - echo "Version unique components:" - def String versionuniquecomponent = "" + def String versionuniquecomponent = '' + def String versionwithpathscomponent = '' for (key in componentReleaseInfoVersionUnique.keySet()) { versionuniquecomponent += "${key}\n" + versionwithpathscomponent = "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" } + echo 'Version unique components:' echo versionuniquecomponent + echo 'Version unique components with paths:' + echo versionwithpathscomponent + if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" From 9d4b6ecf836405b1e75247a29909f86cdddb98eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 14:13:12 +0000 Subject: [PATCH 067/102] Alter release process to handle new meta walker info --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 87eb9ae..c61f1dd 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -416,7 +416,7 @@ def releasehierarchy(String buildfolder, String metafolder, buildmeta, String ma def String versionwithpathscomponent = '' for (key in componentReleaseInfoVersionUnique.keySet()) { versionuniquecomponent += "${key}\n" - versionwithpathscomponent = "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" + versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" } echo 'Version unique components:' echo versionuniquecomponent From 29f666075f68d3769ac2149952d8f74859683eee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 14:22:30 +0000 Subject: [PATCH 068/102] Alter release process to handle new meta walker info --- meta/folder.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index fb27e64..39b8d85 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1043,7 +1043,9 @@ class MetaInfo extends GISJsonSerialised { Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.accumulatedpath.size() > metaitem.accumulatedpath.size()) { + def existingentrypathlength = existingentry.accumulatedpath.split('/').size() + def newentrypathlength = metaitem.accumulatedpath.split('/').size() + if (existingentrypathlength > newentrypathlength) { releasemetacollection.put(uniquestring, metaitem) } } else { From 5e1a008d8f043c3d0ad9309ed7a34734f742a078 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 15:12:33 +0000 Subject: [PATCH 069/102] Alter release process to handle new meta walker info --- check/do_meta.groovy | 6 +++--- differences/dometahierarchy.groovy | 8 ++++---- publish/do.groovy | 4 ++-- publish/publishitems.groovy | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 7bfe2d1..4b8a89e 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = entry.meta.componentversionuniqueidentifier() - def offsetpath = entry.offsetpath + def name = entry.metainfo.componentversionuniqueidentifier() + def offsetpath = entry.accumulatedpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } @@ -65,7 +65,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo2 = componentReleaseInfo[key] - def offsetpath = buildinfo2.offsetpath + def offsetpath = buildinfo2.accumulatedpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6280960..40394d5 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -95,8 +95,8 @@ def showmetaitems(metaitems) { String info = '' for (key in metaitems.keySet()) { def relatedmetainfo = metaitems[key] - def offsetpath = relatedmetainfo.offsetpath - info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" + def accumulatedpath = relatedmetainfo.accumulatedpath + info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" } echo info } @@ -768,8 +768,8 @@ def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" - def earlymeta = earlymetaitems[key].meta - def latermeta = latermetaitems[key].meta + def earlymeta = earlymetaitems[key].metainfo + def latermeta = latermetaitems[key].metainfo summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") diff --git a/publish/do.groovy b/publish/do.groovy index 1b227f4..9f5bd21 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -60,7 +60,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { summaryappend("Artifacts: ") - def releasemeta = releaseinfo.meta + def releasemeta = releaseinfo.metainfo def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo @@ -113,7 +113,7 @@ def copydirectory(File dirFrom, File dirTo) { } def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { - def releasemeta = releaseinfo.meta + def releasemeta = releaseinfo.metainfo def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") diff --git a/publish/publishitems.groovy b/publish/publishitems.groovy index 677970e..06e5fba 100644 --- a/publish/publishitems.groovy +++ b/publish/publishitems.groovy @@ -89,7 +89,7 @@ class PublishableItemsSelection { } def outputDescription(menuitems, releaseinfo) { - def meta = releaseinfo.meta + def meta = releaseinfo.metainfo for (publishableitem in meta.publishableinfo) { menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") @@ -97,7 +97,7 @@ class PublishableItemsSelection { } def outputSelection(menuitems, releaseinfo) { - def meta = releaseinfo.meta + def meta = releaseinfo.metainfo for (publishableitem in meta.publishableinfo) { menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") From 2a7e94b301eccd0dd89c159bf457a75768e1c436 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 16:30:27 +0000 Subject: [PATCH 070/102] Debug components --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 39b8d85..d5b6b7f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1089,6 +1089,7 @@ class MetaInfo extends GISJsonSerialised { ] def walkresults = allrelatedmeta() for (dependentmetaitem in walkresults.metacollection) { + script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") if (dependentmetaitem.metainfo.productinfo.productname == productname && dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") From d945efb8b76503a38d106fbe4f270e7b8c17c5ef Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 16:53:24 +0000 Subject: [PATCH 071/102] Help the meta walker find the meta data for a child --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d5b6b7f..c3db541 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1029,9 +1029,9 @@ class MetaInfo extends GISJsonSerialised { return [ 'metacollection' : results.metacollection, 'comment': results.comment ] } - def allrelatedmeta() { + def allrelatedmeta(String metapath) { ArrayList metacollection = new ArrayList() - return walkrelated(metacollection) + return walkrelated(metacollection, '', 0, '', metapath) } ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { @@ -1082,12 +1082,12 @@ class MetaInfo extends GISJsonSerialised { return new MetaInfo(script) } - def findAssociatedMeta(String productname, String subcomponentname) { + def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { def results = [ 'matchedmetaitem' : new CollectedMeta(this), 'matchfound' : false ] - def walkresults = allrelatedmeta() + def walkresults = allrelatedmeta(parentmetapath) for (dependentmetaitem in walkresults.metacollection) { script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") if (dependentmetaitem.metainfo.productinfo.productname == productname && From f554d5bd6ce258ed2f8623efc8d470b2ce0ef528 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 17:01:19 +0000 Subject: [PATCH 072/102] Help the meta walker find the meta data for a child --- import/importer.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 0e2ca17..8ed4263 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -342,12 +342,12 @@ def artifactimport(meta, } // Import the artifacts of a named child component found below parent meta's hierarchy -def artifactimport(meta, parentmeta, String childproduct, String childsubcomponent, importoptions, String destinationfolder) { +def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, String childsubcomponent, importoptions, String destinationfolder) { // Import from parent meta def childmeta = meta.getNewMetaObject() def childmeta2 echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" - def findresults = parentmeta.findAssociatedMeta(childproduct, childsubcomponent) + def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) if (findresults.matchfound) { childmeta = findresults.matchedmeta if (childmeta.releaseinfo.released) { From 16061cfcc6950dcd5f1cd43e3afc8bef707efd3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 17:11:49 +0000 Subject: [PATCH 073/102] Help the meta walker find the meta data for a child --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index c3db541..bffa75f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1031,7 +1031,7 @@ class MetaInfo extends GISJsonSerialised { def allrelatedmeta(String metapath) { ArrayList metacollection = new ArrayList() - return walkrelated(metacollection, '', 0, '', metapath) + return walkrelated(metacollection, '', 0, '', "${metapath}/${METAFOLDERNAME}") } ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { From 40054880d4ab1a4a9acbc54d3d60fca9f62c4c76 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 17:39:58 +0000 Subject: [PATCH 074/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index bffa75f..2b84781 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1020,7 +1020,7 @@ class MetaInfo extends GISJsonSerialised { limiteddepth, maxdepth - 1) } - } else if (relatedmetainfo.format == '0.1') { + } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { def MetaInfo othermeta = relatedmetainfo.meta results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, gatheredpath, gatheredmetapath, limiteddepth, maxdepth - 1) } From d93eb4bc236d9be47c9082cc1fdcf17756d124fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 18:03:19 +0000 Subject: [PATCH 075/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2b84781..1c85658 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -600,10 +600,12 @@ class MetaInfo extends GISJsonSerialised { abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format + private String offsetpath - RelatedMetaInfo(script, format) { + RelatedMetaInfo(script, format, offsetpath) { super(script) this.format = format + this.offsetpath = offsetpath } abstract void deserialiseJson(rawmeta) @@ -612,15 +614,17 @@ class MetaInfo extends GISJsonSerialised { String format() { return format } + + String getoffsetpath() { + return offsetpath + } } class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { - String offsetpath - MetaInfo meta + private MetaInfo meta RelatedMetaInfo_v0_1(script, offsetpath = "", MetaInfo meta = null) { - super(script, '0.1') - this.offsetpath = offsetpath + super(script, '0.1', offsetpath) this.meta = meta } @@ -638,16 +642,16 @@ class MetaInfo extends GISJsonSerialised { ] return data } + + String get } class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { private String importpath - private String offsetpath RelatedMetaInfo_v0_2(script, importpath = "", offsetpath = "") { - super(script, '0.2') + super(script, '0.2', offsetpath) this.importpath = importpath - this.offsetpath = offsetpath } void deserialiseJson(rawmeta) { @@ -1009,6 +1013,7 @@ class MetaInfo extends GISJsonSerialised { results.metacollection.add(new CollectedMeta(this, gatheredpath, gatheredmetapath)) results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' for (relatedmetainfo in associatedmeta) { + script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") if (relatedmetainfo.format == '0.2') { def childmetainfo = openchildmeta(gatheredpath, gatheredmetapath, relatedmetainfo) if (childmetainfo.imported) { From f09c84e5ec6404478eb92ae21c0ed0ad9e0d883e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 18:13:50 +0000 Subject: [PATCH 076/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1c85658..7dd4521 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -600,7 +600,7 @@ class MetaInfo extends GISJsonSerialised { abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format - private String offsetpath + protected String offsetpath RelatedMetaInfo(script, format, offsetpath) { super(script) From f44dc09dab4eb08b89c14298c0cc9032dd3ed355 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 18:28:34 +0000 Subject: [PATCH 077/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7dd4521..cccc456 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -600,7 +600,7 @@ class MetaInfo extends GISJsonSerialised { abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format - protected String offsetpath + private String offsetpath RelatedMetaInfo(script, format, offsetpath) { super(script) @@ -618,6 +618,10 @@ class MetaInfo extends GISJsonSerialised { String getoffsetpath() { return offsetpath } + + void setoffsetoffsetpath(String offsetpath) { + this.offsetpath = offsetpath + } } class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { @@ -629,7 +633,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - offsetpath = rawmeta['offsetpath'] + setoffsetoffsetpath(rawmeta['offsetpath']) meta = new MetaInfo(script) meta.deserialiseJson(rawmeta['meta']) } @@ -637,7 +641,7 @@ class MetaInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ 'format' : format, - 'offsetpath' : offsetpath, + 'offsetpath' : getoffsetpath(), 'meta' : meta.serialisationdata(), ] return data @@ -656,13 +660,13 @@ class MetaInfo extends GISJsonSerialised { void deserialiseJson(rawmeta) { //format = rawmeta['format'] - offsetpath = rawmeta['offsetpath'] + setoffsetoffsetpath(rawmeta['offsetpath']) } LinkedHashMap serialisationdata() { def data = [ 'format' : format, - 'offsetpath' : offsetpath, + 'offsetpath' : getoffsetpath(), ] return data } From cd43ecf33aba1e6c91e492ac0984cc1b4069620e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 17 Dec 2021 18:37:18 +0000 Subject: [PATCH 078/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cccc456..712795b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -600,7 +600,7 @@ class MetaInfo extends GISJsonSerialised { abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format - private String offsetpath + String offsetpath RelatedMetaInfo(script, format, offsetpath) { super(script) From 6e3c993d3e0c3f49f6f768a585c4a9864b09d4aa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 18:02:11 +0000 Subject: [PATCH 079/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 712795b..0b6128b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -600,32 +600,26 @@ class MetaInfo extends GISJsonSerialised { abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { private String format - String offsetpath RelatedMetaInfo(script, format, offsetpath) { super(script) this.format = format this.offsetpath = offsetpath } - - abstract void deserialiseJson(rawmeta) - abstract LinkedHashMap serialisationdata() String format() { return format } - String getoffsetpath() { - return offsetpath - } - - void setoffsetoffsetpath(String offsetpath) { - this.offsetpath = offsetpath - } + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + abstract String getoffsetpath() + abstract void setoffsetpath(String offsetpath) } class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { private MetaInfo meta + private String offsetpath RelatedMetaInfo_v0_1(script, offsetpath = "", MetaInfo meta = null) { super(script, '0.1', offsetpath) @@ -633,7 +627,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - setoffsetoffsetpath(rawmeta['offsetpath']) + setoffsetpath(rawmeta['offsetpath']) meta = new MetaInfo(script) meta.deserialiseJson(rawmeta['meta']) } @@ -647,11 +641,18 @@ class MetaInfo extends GISJsonSerialised { return data } - String get + String getoffsetpath() { + return offsetpath + } + + void setoffsetpath(String offsetpath) { + this.offsetpath = offsetpath + } } class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { private String importpath + private String offsetpath RelatedMetaInfo_v0_2(script, importpath = "", offsetpath = "") { super(script, '0.2', offsetpath) @@ -660,7 +661,7 @@ class MetaInfo extends GISJsonSerialised { void deserialiseJson(rawmeta) { //format = rawmeta['format'] - setoffsetoffsetpath(rawmeta['offsetpath']) + setoffsetpath(rawmeta['offsetpath']) } LinkedHashMap serialisationdata() { @@ -670,6 +671,14 @@ class MetaInfo extends GISJsonSerialised { ] return data } + + String getoffsetpath() { + return offsetpath + } + + void setoffsetpath(String offsetpath) { + this.offsetpath = offsetpath + } } private ProductInfo productinfo From 7e63a317c07b99a77ac76e3f3875f9601ee78388 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 18:21:30 +0000 Subject: [PATCH 080/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0b6128b..76dae26 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -599,16 +599,11 @@ class MetaInfo extends GISJsonSerialised { } abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { - private String format + String format - RelatedMetaInfo(script, format, offsetpath) { + RelatedMetaInfo(script, format) { super(script) this.format = format - this.offsetpath = offsetpath - } - - String format() { - return format } abstract void deserialiseJson(rawmeta) @@ -622,7 +617,8 @@ class MetaInfo extends GISJsonSerialised { private String offsetpath RelatedMetaInfo_v0_1(script, offsetpath = "", MetaInfo meta = null) { - super(script, '0.1', offsetpath) + super(script, '0.1') + this.offsetpath = offsetpath this.meta = meta } @@ -655,8 +651,9 @@ class MetaInfo extends GISJsonSerialised { private String offsetpath RelatedMetaInfo_v0_2(script, importpath = "", offsetpath = "") { - super(script, '0.2', offsetpath) + super(script, '0.2') this.importpath = importpath + this.offsetpath = offsetpath } void deserialiseJson(rawmeta) { From 95506b8d8a6f045ce727e5a6c6c1c85f33b63f3d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 18:26:02 +0000 Subject: [PATCH 081/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 76dae26..fc1e072 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -595,7 +595,8 @@ class MetaInfo extends GISJsonSerialised { interface IRelatedMetaInfo { abstract void deserialiseJson(rawmeta) abstract LinkedHashMap serialisationdata() - abstract String format() + abstract String getoffsetpath() + abstract void setoffsetpath(String offsetpath) } abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { From 25938cefe32f899ef876546625fe6688d7381d4c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 18:34:36 +0000 Subject: [PATCH 082/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index fc1e072..7016b0b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -595,8 +595,6 @@ class MetaInfo extends GISJsonSerialised { interface IRelatedMetaInfo { abstract void deserialiseJson(rawmeta) abstract LinkedHashMap serialisationdata() - abstract String getoffsetpath() - abstract void setoffsetpath(String offsetpath) } abstract class RelatedMetaInfo extends GISJsonSerialised implements IRelatedMetaInfo { @@ -609,8 +607,6 @@ class MetaInfo extends GISJsonSerialised { abstract void deserialiseJson(rawmeta) abstract LinkedHashMap serialisationdata() - abstract String getoffsetpath() - abstract void setoffsetpath(String offsetpath) } class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { From c76a1d68e51ad427a8f55871f93d50a2f299656e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 18:51:19 +0000 Subject: [PATCH 083/102] Handle v0.1 imported associated meta --- meta/folder.groovy | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7016b0b..1919d1b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -620,7 +620,7 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - setoffsetpath(rawmeta['offsetpath']) + offsetpath = rawmeta['offsetpath'] meta = new MetaInfo(script) meta.deserialiseJson(rawmeta['meta']) } @@ -628,19 +628,11 @@ class MetaInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ 'format' : format, - 'offsetpath' : getoffsetpath(), + 'offsetpath' : offsetpath, 'meta' : meta.serialisationdata(), ] return data } - - String getoffsetpath() { - return offsetpath - } - - void setoffsetpath(String offsetpath) { - this.offsetpath = offsetpath - } } class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { @@ -655,24 +647,16 @@ class MetaInfo extends GISJsonSerialised { void deserialiseJson(rawmeta) { //format = rawmeta['format'] - setoffsetpath(rawmeta['offsetpath']) + offsetpath = rawmeta['offsetpath'] } LinkedHashMap serialisationdata() { def data = [ 'format' : format, - 'offsetpath' : getoffsetpath(), + 'offsetpath' : offsetpath, ] return data } - - String getoffsetpath() { - return offsetpath - } - - void setoffsetpath(String offsetpath) { - this.offsetpath = offsetpath - } } private ProductInfo productinfo From 670d7b22abadc922ada2f8e80538c7a3ada56661 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 19:22:20 +0000 Subject: [PATCH 084/102] Handle v0.1 imported associated meta --- import/importer.groovy | 2 +- meta/folder.groovy | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 8ed4263..39b31c6 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -349,7 +349,7 @@ def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) if (findresults.matchfound) { - childmeta = findresults.matchedmeta + childmeta = findresults.matchedmetaitem.metainfo if (childmeta.releaseinfo.released) { importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) } else { diff --git a/meta/folder.groovy b/meta/folder.groovy index 1919d1b..e2a4cbd 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1018,7 +1018,13 @@ class MetaInfo extends GISJsonSerialised { } } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { def MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, gatheredpath, gatheredmetapath, limiteddepth, maxdepth - 1) + results = othermeta.walkrelated_v0_1(results.metacollection, + results.comment, + level + 1, + gatheredpath, + gatheredmetapath, + limiteddepth, + maxdepth - 1) } } } From 8e53968fbdfa588507b4d93c5a7f96e30d6077b6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 20:38:30 +0000 Subject: [PATCH 085/102] Type parameters to archive method --- artifacts/archiver.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy index 78bee55..bafba02 100644 --- a/artifacts/archiver.groovy +++ b/artifacts/archiver.groovy @@ -6,7 +6,7 @@ enum Store { this.JENKINS = Store.JENKINS this.ARTIFACTSERVER = Store.JENKINS -def archive(meta, artifactlist, Store store = JENKINS ) { +def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { echo "Archiving artifacts to store: ${store}" meta.copyrelatedmeta() if (store == JENKINS) { From 1a111411353902deb9fc6ba76824559c501a2a4e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 18 Dec 2021 21:03:34 +0000 Subject: [PATCH 086/102] Type parameters to archive method --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index e2a4cbd..9ddeb7c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -872,10 +872,10 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here - ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here - String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" - ) { + def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { def String artifactlist = METAARTIFACTS releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { From 0a0df4d88a2f7608002bb06ba2d21adc851017a7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 13:23:20 +0000 Subject: [PATCH 087/102] Load module before cleaning --- copy/all.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/all.groovy b/copy/all.groovy index 9b95eb6..0b7d862 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -54,6 +54,7 @@ pipeline { stage('Archiving artifacts') { steps { script { + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") def includes = '**/*' def excludes = "${IMPORTS}/*" echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" @@ -65,7 +66,6 @@ pipeline { 'autobuild', ] cleaner.clean(cleanfolders) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") artifactarchiver.archive(artifacts: includes) } } From ab0d724399e4f476f584af9bfab18b40ba60c184 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 13:39:47 +0000 Subject: [PATCH 088/102] Load module before cleaning --- copy/all.groovy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/copy/all.groovy b/copy/all.groovy index 0b7d862..920c80b 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -54,19 +54,15 @@ pipeline { stage('Archiving artifacts') { steps { script { - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") def includes = '**/*' def excludes = "${IMPORTS}/*" echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" - // This does not work: artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - // This does not work: artifactarchiver.archive(artifacts: includes, excludes: excludes) - // so cleanfolders = [ IMPORTS, 'autobuild', ] cleaner.clean(cleanfolders) - artifactarchiver.archive(artifacts: includes) + archiveArtifacts(artifacts: includes) } } } From 07a8a9d6a3059f250b058b7cf1e41e97dae7e625 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 18:21:23 +0000 Subject: [PATCH 089/102] Use indirect artifact archiver --- copy/all.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/copy/all.groovy b/copy/all.groovy index 920c80b..518f593 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -54,15 +54,19 @@ pipeline { stage('Archiving artifacts') { steps { script { - def includes = '**/*' + def artifacts = ['**/*'] + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + def artifactlist = meta.generateartifacts(artifacts) def excludes = "${IMPORTS}/*" - echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" + echo "Archiving artifacts: Includes: ${artifacts} Excludes: ${excludes}" + cleanfolders = [ IMPORTS, 'autobuild', ] cleaner.clean(cleanfolders) - archiveArtifacts(artifacts: includes) + + artifactarchiver.archive(meta, artifactlist) } } } From 6b35583fd2565683a197829f481c5521aef0af11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 21:13:30 +0000 Subject: [PATCH 090/102] Use clean while moving imported info --- copy/all.groovy | 31 +++++++++++-------------------- copy/logic.groovy | 39 +++++++++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/copy/all.groovy b/copy/all.groovy index 518f593..aab479c 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -5,6 +5,8 @@ pipeline { steps { script { meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") } } } @@ -23,7 +25,6 @@ pipeline { stage('Import source artifacts') { steps { script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def copyoptions = importer.getNewImportOptions() copyoptions.source( SOURCE_ARTIFACT_LOCATION, @@ -37,9 +38,14 @@ pipeline { } } } - stage('Copy source artifacts') { + stage('Archiving artifacts') { steps { script { + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + dir(IMPORTS) { fileOperations([ fileCopyOperation(excludes: '', @@ -48,23 +54,8 @@ pipeline { targetLocation: WORKSPACE) ]) } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifacts = ['**/*'] - def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - def artifactlist = meta.generateartifacts(artifacts) - def excludes = "${IMPORTS}/*" - echo "Archiving artifacts: Includes: ${artifacts} Excludes: ${excludes}" - - cleanfolders = [ - IMPORTS, - 'autobuild', - ] - cleaner.clean(cleanfolders) + + cleaner.clean([IMPORTS]) artifactarchiver.archive(meta, artifactlist) } @@ -85,4 +76,4 @@ pipeline { echo 'Succeeded build' } } -} \ No newline at end of file +} diff --git a/copy/logic.groovy b/copy/logic.groovy index 0a28a22..bf90a71 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -13,22 +13,25 @@ pipeline { stage('Clean exports') { steps { script { - env.LOGICDEST = "${WORKSPACE}/logic" + LOGICDEST = 'logic' cleanfolders = [ LOGICDEST, - "${WORKSPACE}/embedded_logic", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Copying logic artifacts') { + stage('Import source artifacts') { steps { script { - sh('env') def logicoptions = importer.getNewImportOptions() - logicoptions.source(LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_ARTIFACT_RELEASE_REPO, LOGIC_ARTIFACT_RELEASE_REVISION) + logicoptions.source( + LOGIC_ARTIFACT_LOCATION, + LOGIC_ARTIFACT_PROJECT, + LOGIC_ARTIFACT_BUILD_SELECTOR, + LOGIC_ARTIFACT_RELEASE_REPO, + LOGIC_ARTIFACT_RELEASE_REVISION) importer.artifactimport(meta, logicoptions, LOGICDEST) } } @@ -36,14 +39,22 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["embedded_logic/**"]) - sh """ - rm -v -rf meta - # The following is removing a deprecated folder - rm -v -rf diff - mv -v "logic"/* . - """ - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + + dir(LOGICDEST) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + + cleaner.clean([LOGICDEST]) + artifactarchiver.archive(meta, artifactlist) } } @@ -63,4 +74,4 @@ pipeline { echo 'Succeeded build' } } -} \ No newline at end of file +} From c3c7b802938939e255c81b15aa33c72fea169da0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 21:43:27 +0000 Subject: [PATCH 091/102] Use clean while moving imported info --- copy/all.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/copy/all.groovy b/copy/all.groovy index aab479c..37a47e6 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -45,6 +45,8 @@ pipeline { def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") cleaner.clean(['meta', 'autobuild']) + + error('Debugging') dir(IMPORTS) { fileOperations([ From a06ac842b9a4ca4c03469eac9a7d56e3c6bbcf4d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 21:45:33 +0000 Subject: [PATCH 092/102] Use clean while moving imported info --- copy/all.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/copy/all.groovy b/copy/all.groovy index 37a47e6..486a627 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -45,8 +45,6 @@ pipeline { def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") cleaner.clean(['meta', 'autobuild']) - - error('Debugging') dir(IMPORTS) { fileOperations([ @@ -57,6 +55,7 @@ pipeline { ]) } + error('Debugging') cleaner.clean([IMPORTS]) artifactarchiver.archive(meta, artifactlist) From d7893d9160ac5b0388a7f56f55a447ff05b67021 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 19 Dec 2021 21:53:41 +0000 Subject: [PATCH 093/102] Use clean while moving imported info --- copy/all.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/copy/all.groovy b/copy/all.groovy index 486a627..aab479c 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -55,7 +55,6 @@ pipeline { ]) } - error('Debugging') cleaner.clean([IMPORTS]) artifactarchiver.archive(meta, artifactlist) From c9eb0c698c80a7e8c3c20cc1f9b3b10bbdc5e0f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 13:52:31 +0000 Subject: [PATCH 094/102] Don't create folders for imported artifacts that don't have meta --- meta/folder.groovy | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9ddeb7c..6e6767b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -825,16 +825,15 @@ class MetaInfo extends GISJsonSerialised { if (script.fileExists(importedmetafile)) { script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) + + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } } else { script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - - RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath) - if (appendtoparentmeta) { - associatedmeta.add(rmi) - } - return mi } From 01d66c2d893609b513a78f7d75dedb7ac6820d99 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 14:47:22 +0000 Subject: [PATCH 095/102] Don't create folders for imported artifacts that don't have meta --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6e6767b..f43237c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -832,7 +832,7 @@ class MetaInfo extends GISJsonSerialised { } } else { script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") - mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) + addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } return mi } From d3e5eb91acf81b01723fbc42111e1321b6f92461 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 15:55:07 +0000 Subject: [PATCH 096/102] Correct build integrity checking --- meta/folder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f43237c..96a37f8 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1030,9 +1030,11 @@ class MetaInfo extends GISJsonSerialised { return [ 'metacollection' : results.metacollection, 'comment': results.comment ] } - def allrelatedmeta(String metapath) { + def allrelatedmeta(String metapath = '') { ArrayList metacollection = new ArrayList() - return walkrelated(metacollection, '', 0, '', "${metapath}/${METAFOLDERNAME}") + + def startmetapath = (metapath == '' ? METAFOLDERNAME : "${metapath}/${METAFOLDERNAME}") + return walkrelated(metacollection, '', 0, '', startmetapath) } ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { From 09b3c99bfe52c44397a813d8373cc80fa7d64bfd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 16:46:50 +0000 Subject: [PATCH 097/102] Correct initial gather paths --- meta/folder.groovy | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 96a37f8..e6bd3c5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -923,18 +923,8 @@ class MetaInfo extends GISJsonSerialised { def openchildmeta(String gatheredpath, String gatheredmetapath, RelatedMetaInfo relatedmetainfo) { def MetaInfo mi = new MetaInfo(script) - def String nextgatheredpath - if (gatheredpath == '') { - nextgatheredpath = "${relatedmetainfo.offsetpath}" - } else { - nextgatheredpath = "${gatheredpath}/${relatedmetainfo.offsetpath}" - } - def String nextgatheredmetapath - if (gatheredmetapath == '') { - nextgatheredmetapath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" - } else { - nextgatheredmetapath = "${gatheredmetapath}/${relatedmetainfo.offsetpath}" - } + def String nextgatheredpath = (gatheredpath == '' ? relatedmetainfo.offsetpath : "${gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${gatheredmetapath}/${relatedmetainfo.offsetpath}" def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" def boolean imported if (script.fileExists(importedmetafile)) { @@ -972,11 +962,13 @@ class MetaInfo extends GISJsonSerialised { for (relatedmetainfo in associatedmeta) { if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (gatheredpath == '' ? relatedmetainfo.offsetpath : "${gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (gatheredmetapath == '' ? gatheredmetapath : "${gatheredmetapath}/${relatedmetainfo.offsetpath}") results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, - "${gatheredpath}/${relatedmetainfo.offsetpath}", - "${gatheredmetapath}/${relatedmetainfo.offsetpath}", + nextgatheredpath, + nextgatheredmetapath, limiteddepth, maxdepth - 1) } else { @@ -1032,9 +1024,7 @@ class MetaInfo extends GISJsonSerialised { def allrelatedmeta(String metapath = '') { ArrayList metacollection = new ArrayList() - - def startmetapath = (metapath == '' ? METAFOLDERNAME : "${metapath}/${METAFOLDERNAME}") - return walkrelated(metacollection, '', 0, '', startmetapath) + return walkrelated(metacollection, '', 0, '', metapath) } ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { From f171e520b7685d35acfd605b3f96dd99dc0cc946 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 16:47:52 +0000 Subject: [PATCH 098/102] Correct initial gather paths --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index e6bd3c5..aa1d673 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -924,7 +924,7 @@ class MetaInfo extends GISJsonSerialised { def MetaInfo mi = new MetaInfo(script) def String nextgatheredpath = (gatheredpath == '' ? relatedmetainfo.offsetpath : "${gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${gatheredmetapath}/${relatedmetainfo.offsetpath}" + def String nextgatheredmetapath = (gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${gatheredmetapath}/${relatedmetainfo.offsetpath}") def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" def boolean imported if (script.fileExists(importedmetafile)) { From 6053dbc4a93c847070dfde3524cdc10a95413956 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 17:27:32 +0000 Subject: [PATCH 099/102] Correct initial gather paths --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index aa1d673..883e690 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -963,7 +963,7 @@ class MetaInfo extends GISJsonSerialised { if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { def MetaInfo othermeta = relatedmetainfo.meta def String nextgatheredpath = (gatheredpath == '' ? relatedmetainfo.offsetpath : "${gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (gatheredmetapath == '' ? gatheredmetapath : "${gatheredmetapath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${gatheredmetapath}/${relatedmetainfo.offsetpath}") results = othermeta.walkrelated_v0_1(results.metacollection, results.comment, level + 1, From d5ecbe17d4948da2a8103046d68d269a938d7c8a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 23:01:03 +0000 Subject: [PATCH 100/102] Add log --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index 39b31c6..0e88a21 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -347,6 +347,7 @@ def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, def childmeta = meta.getNewMetaObject() def childmeta2 echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" + echo "From Parent meta identifier: ${parentmeta.componentidentifier} and parent meta path: ${parentmetapath}" def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) if (findresults.matchfound) { childmeta = findresults.matchedmetaitem.metainfo From af70f7e083ecdec2812ce86b19d3245368b13612 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 23:04:05 +0000 Subject: [PATCH 101/102] Add log --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 0e88a21..a35598c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -347,7 +347,7 @@ def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, def childmeta = meta.getNewMetaObject() def childmeta2 echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" - echo "From Parent meta identifier: ${parentmeta.componentidentifier} and parent meta path: ${parentmetapath}" + echo "From Parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) if (findresults.matchfound) { childmeta = findresults.matchedmetaitem.metainfo From 4b3e84c72d851b2ed499558d02b72dc26aa41f23 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 Dec 2021 23:14:41 +0000 Subject: [PATCH 102/102] Alter path to start finding --- import/importer.groovy | 2 +- meta/folder.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index a35598c..8bb565a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -347,7 +347,7 @@ def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, def childmeta = meta.getNewMetaObject() def childmeta2 echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" - echo "From Parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" + echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) if (findresults.matchfound) { childmeta = findresults.matchedmetaitem.metainfo diff --git a/meta/folder.groovy b/meta/folder.groovy index 883e690..f72b502 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1080,7 +1080,7 @@ class MetaInfo extends GISJsonSerialised { 'matchedmetaitem' : new CollectedMeta(this), 'matchfound' : false ] - def walkresults = allrelatedmeta(parentmetapath) + def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") for (dependentmetaitem in walkresults.metacollection) { script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") if (dependentmetaitem.metainfo.productinfo.productname == productname &&