From c2582fe710abe4d307c5fde5eb17f02a4a6cfd67 Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 28 Oct 2019 11:53:49 +0000 Subject: [PATCH 01/54] Updated scripts to refer to SI1536 head. --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 5c0f15d..2b7f589 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -83,6 +83,9 @@ jiraprojectpattern = { 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), ), + 'HMBSI1536': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ss][Ii]1536[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBSI1536-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From 958ad765d03041dbe18bf4e80b1697bdeddb7aa3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 12 Dec 2019 11:32:56 +0000 Subject: [PATCH 02/54] GIS-1621 - Handle FD1200 embedded software building --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index 25b2614..a8b64f5 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -106,6 +106,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { switch (source) { case "Jenkins job artifact": + case "Jenkins job artefact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) case "Release repository": if (nested) { From 21c59682a1976ee78a41222bdbcbdd27be736866 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 13 Dec 2019 14:32:32 +0000 Subject: [PATCH 03/54] GIS-1621 - Make an ESW copier job script --- copy/esw.groovy | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 copy/esw.groovy diff --git a/copy/esw.groovy b/copy/esw.groovy new file mode 100644 index 0000000..2a63171 --- /dev/null +++ b/copy/esw.groovy @@ -0,0 +1,63 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + env.ESW = 'esw' + env.ESWDEST = "${WORKSPACE}/${ESW}" + env.ESWEXPORT = 'embedded_software' + cleanfolders = [ + ESWDEST, + ESWEXPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying embedded software artifacts') { + steps { + script { + importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) + sh """ + rm -v -rf diff + rm -v -rf meta + mv -v "${ESW}"/* . + """ + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} \ No newline at end of file From 2241c0c982e1b9a5a36fbd2660b8f9de99bc4631 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 17 Dec 2019 18:38:57 +0000 Subject: [PATCH 04/54] GIS-1621 - Allow logic copies from repositories --- copy/logic.groovy | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 4b3d521..8c82d02 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -5,16 +5,18 @@ pipeline { steps { script { meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") } } } stage('Clean exports') { steps { script { + env.LOGICDEST = "${WORKSPACE}/logic" cleanfolders = [ - "${WORKSPACE}/logic", + LOGICDEST, "${WORKSPACE}/embedded_logic", - "${WORKSPACE}/diff", // Some old artifacts have this folder ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -24,11 +26,7 @@ pipeline { stage('Copying logic artifacts') { steps { script { - env.LOGICDEST = "${WORKSPACE}/logic" - if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) - } + importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC _RELEASE_REVISION, LOGICDEST) } } } @@ -37,7 +35,6 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf diff rm -v -rf meta mv -v "logic"/* . """ From d0f9244af4c83124b338a54c8ac36ece48973290 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 17 Dec 2019 18:43:14 +0000 Subject: [PATCH 05/54] GIS-1621 - Allow logic copies from repositories --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 8c82d02..4d5291c 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -26,7 +26,7 @@ pipeline { stage('Copying logic artifacts') { steps { script { - importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC _RELEASE_REVISION, LOGICDEST) + importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC_RELEASE_REVISION, LOGICDEST) } } } From 0ea84ea80eaa0427f39b6a8e067bd194d66516e0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 19 Dec 2019 08:57:51 +0000 Subject: [PATCH 06/54] GIS-1621 - Correct script --- staticanalysis/ccm.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index 93e4711..865e5fa 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -66,7 +66,7 @@ pipeline { echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" if (isUnix()) { sh """#!/bin/bash - find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \; + find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \\; # Uses mono "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml """ From 83e8b67ca601e874fe2c7abad5debc9351c5a347 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:04:34 +0000 Subject: [PATCH 07/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 218 +++++++++++++++++++++++++++++++++-------- 1 file changed, 179 insertions(+), 39 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index a8b64f5..666f389 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,36 +1,122 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { - copyArtifacts( - filter: filter, - excludes: excludes, - fingerprintArtifacts: fingerprint, - projectName: jenkinsjob.trim(), - selector: specific(buildnumber.toString()), - target: destpath - ) - def importedmeta = meta.importrelated(destpath) - if (incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) +class ImportOptions { + def script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + boolean incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String filter // Only import files/paths/folders that match the filter + String excludes // Exclude files/paths/folders that match the exclude filter + boolean fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean nested // Import all child artifacts too (identified in the related meta info + boolean appendtoparentmeta // Append to parent meta + String source // Where are the artifacts coming from + + String jenkinsjob + def buildnumberorselector + + String repo + String revision + + ImportOptions(script, + String source = 'Jenkins job artifact' + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false, + appendtoparentmeta = true) { + this.script = script + setsource(source) + setIncrementVersionNumber(incrementversionnumber) + setfilter(filter) + setexcludes(excludes) + setfingerprint(fingerprint) + setnested(nested) + setappendtoparentmeta(appendtoparentmeta) + } + + setsource(String source) { + source = source.trim() + switch (source) { + case 'Jenkins job artefact': + this.source = 'Jenkins job artifact' + break + case 'Jenkins job artifact': + case 'Release repository': + this.source = source + break + default: + error("Unknown type of artifact source: ${source}") + } + } + + setIncrementVersionNumber(boolean incrementversionnumber) { + this.incrementversionnumber = incrementversionnumber + } + + setfilter(String filter) { + this.filter = filter.trim() + } + + setexcludes(String excludes) { + this.excludes = excludes.trim() + } + + setfingerprint(boolean fingerprint) { + this.fingerprint = fingerprint + } + + setnested(boolean nested) { + this.nested = nested + } + + setappendtoparentmeta(boolean appendtoparentmeta) { + this.appendtoparentmeta = appendtoparentmeta + } + + jenkinsjob(String jenkinsjob, buildnumberorselector) { + this.jenkinsjob = jenkinsjob.trim() + this.buildnumberorselector = buildnumberorselector.trim() + setsource('Jenkins job artifact') + } + + releaserepo(String repo, String revision) { + this.repo = repo.trim() + this.revision = revision.trim() + setsource('Release repository') + } + + source(String source, String jenkinsjob, buildnumberorselector, String repo, String revision) { + jenkinsjob(String jenkinsjob, buildnumberorselector) + releaserepo(String repo, String revision) + setsource(source) } - return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { +ImportOptions getNewImportOptions(script) { + return new ImportOptions(script) +} + +def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { + if (importoptions.source != 'Jenkins job artifact') { + error("Import options settings have not been set for: Jenkins job artifact") + return + } + copyArtifacts( - filter: filter, - excludes: excludes, + filter: importoptions.filter, + excludes: importoptions.excludes, fingerprintArtifacts: fingerprint, - projectName: jenkinsjob.trim(), + projectName: importoptions.jenkinsjob, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: buildselector + parameterName: importoptions.buildselector ], target: destpath ) def importedmeta = meta.importrelated(destpath) - if (incrementversionnumber) { + if (importoptions.incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta @@ -42,13 +128,17 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") -def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { - def rooturl = repo.trim() - def rootrevision = revision.trim() +def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { + if (importoptions.source != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + def rooturl = importoptions.repo + def rootrevision = importoptions.revision echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) @@ -76,18 +166,23 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - if (incrementversionnumber) { + if (importoptions.incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { - def rooturl = repo.trim() - def rootrevision = revision.trim() +def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { + if (importoptions.source != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + def rooturl = importoptions.repo + def rootrevision = importoptions.revision echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) @@ -97,26 +192,71 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, echo "Marking meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - if (incrementversionnumber) { + if (importoptions.incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { - switch (source) { - case "Jenkins job artifact": - case "Jenkins job artefact": - return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) - case "Release repository": - if (nested) { - return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) +def artifactimport(meta, ImportOptions importoptions, String destpath) { + switch (importoptions.source) { + case 'Jenkins job artifact': + case 'Jenkins job artefact': + return artifactimportfrombuildjob(meta, importoptions, destpath) + case 'Release repository': + if (importoptions.nested) { + return hierarchyartifactimportfromrepo(meta, importoptions, destpath) } - return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + return artifactimportfromrepo(meta, importoptions, destpath) default: error("Unknown type of artifact source: ${source}") } return meta } +// Deprecated (use above) +def artifactimport(meta, + String source, + String jenkinsjob, + buildnumberorselector, + String repo, + String revision, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false) { + importoptions = getNewImportOptions(this) + + importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + importoptions.setnested(nested) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String jenkinsjob, + int buildnumber, + String destpath, + boolean incrementversionnumber = true, + String filter = '**', + String excludes = '', + boolean fingerprint = false) { + importoptions = getNewImportOptions(this) + + importoptions.source(jenkinsjob, specific(buildnumber.toString()) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + return this; From 2590912804b3fd354dc3367b434368ff70457c9d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:07:29 +0000 Subject: [PATCH 08/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 666f389..fc844fb 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -19,7 +19,7 @@ class ImportOptions { String revision ImportOptions(script, - String source = 'Jenkins job artifact' + String source = 'Jenkins job artifact', boolean incrementversionnumber = true, String filter = '', String excludes = '', From 6b6a6fb8e7df309cb3b59b3dc01af73e89e8e649 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:09:24 +0000 Subject: [PATCH 09/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index fc844fb..6ca3c53 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -36,7 +36,7 @@ class ImportOptions { setappendtoparentmeta(appendtoparentmeta) } - setsource(String source) { + void setsource(String source) { source = source.trim() switch (source) { case 'Jenkins job artefact': @@ -51,43 +51,43 @@ class ImportOptions { } } - setIncrementVersionNumber(boolean incrementversionnumber) { + void setIncrementVersionNumber(boolean incrementversionnumber) { this.incrementversionnumber = incrementversionnumber } - setfilter(String filter) { + void setfilter(String filter) { this.filter = filter.trim() } - setexcludes(String excludes) { + void setexcludes(String excludes) { this.excludes = excludes.trim() } - setfingerprint(boolean fingerprint) { + void setfingerprint(boolean fingerprint) { this.fingerprint = fingerprint } - setnested(boolean nested) { + void setnested(boolean nested) { this.nested = nested } - setappendtoparentmeta(boolean appendtoparentmeta) { + void setappendtoparentmeta(boolean appendtoparentmeta) { this.appendtoparentmeta = appendtoparentmeta } - jenkinsjob(String jenkinsjob, buildnumberorselector) { + void jenkinsjob(String jenkinsjob, buildnumberorselector) { this.jenkinsjob = jenkinsjob.trim() this.buildnumberorselector = buildnumberorselector.trim() setsource('Jenkins job artifact') } - releaserepo(String repo, String revision) { + void releaserepo(String repo, String revision) { this.repo = repo.trim() this.revision = revision.trim() setsource('Release repository') } - source(String source, String jenkinsjob, buildnumberorselector, String repo, String revision) { + void source(String source, String jenkinsjob, buildnumberorselector, String repo, String revision) { jenkinsjob(String jenkinsjob, buildnumberorselector) releaserepo(String repo, String revision) setsource(source) From 1156a90c2d7406e98b61b6890503d0834d2725e5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:10:53 +0000 Subject: [PATCH 10/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 6ca3c53..7f787dc 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -250,7 +250,7 @@ def artifactimport(meta, boolean fingerprint = false) { importoptions = getNewImportOptions(this) - importoptions.source(jenkinsjob, specific(buildnumber.toString()) + importoptions.source(jenkinsjob, specific(buildnumber.toString())) importoptions.setincrementversionnumber(incrementversionnumber) importoptions.setfilter(filter) importoptions.setexcludes(excludes) From f6f42b5f1bd00b6a2030d798ee678ee1c1f8b4a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:11:55 +0000 Subject: [PATCH 11/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 7f787dc..e1ef7c5 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -88,8 +88,8 @@ class ImportOptions { } void source(String source, String jenkinsjob, buildnumberorselector, String repo, String revision) { - jenkinsjob(String jenkinsjob, buildnumberorselector) - releaserepo(String repo, String revision) + jenkinsjob(jenkinsjob, buildnumberorselector) + releaserepo(repo, revision) setsource(source) } } From 9f0f554db7c6ac0b389dfdec796320d8361909d0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:20:16 +0000 Subject: [PATCH 12/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index e1ef7c5..d51379a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -36,6 +36,7 @@ class ImportOptions { setappendtoparentmeta(appendtoparentmeta) } + @NonCPS void setsource(String source) { source = source.trim() switch (source) { From ec153471ea774d4d91d0e271c0b8ab563f052043 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:26:26 +0000 Subject: [PATCH 13/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index d51379a..2c9daee 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -52,42 +52,51 @@ class ImportOptions { } } + @NonCPS void setIncrementVersionNumber(boolean incrementversionnumber) { this.incrementversionnumber = incrementversionnumber } + @NonCPS void setfilter(String filter) { this.filter = filter.trim() } + @NonCPS void setexcludes(String excludes) { this.excludes = excludes.trim() } + @NonCPS void setfingerprint(boolean fingerprint) { this.fingerprint = fingerprint } + @NonCPS void setnested(boolean nested) { this.nested = nested } - + + @NonCPS void setappendtoparentmeta(boolean appendtoparentmeta) { this.appendtoparentmeta = appendtoparentmeta } + @NonCPS void jenkinsjob(String jenkinsjob, buildnumberorselector) { this.jenkinsjob = jenkinsjob.trim() this.buildnumberorselector = buildnumberorselector.trim() setsource('Jenkins job artifact') } + @NonCPS void releaserepo(String repo, String revision) { this.repo = repo.trim() this.revision = revision.trim() setsource('Release repository') } + @NonCPS void source(String source, String jenkinsjob, buildnumberorselector, String repo, String revision) { jenkinsjob(jenkinsjob, buildnumberorselector) releaserepo(repo, revision) From fb87ab631e7d7164253e79acf531ed483acc9605 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:31:35 +0000 Subject: [PATCH 14/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 2c9daee..75c4412 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -83,7 +83,7 @@ class ImportOptions { } @NonCPS - void jenkinsjob(String jenkinsjob, buildnumberorselector) { + void jenkinsjob(String jenkinsjob, String buildnumberorselector) { this.jenkinsjob = jenkinsjob.trim() this.buildnumberorselector = buildnumberorselector.trim() setsource('Jenkins job artifact') @@ -97,7 +97,7 @@ class ImportOptions { } @NonCPS - void source(String source, String jenkinsjob, buildnumberorselector, String repo, String revision) { + void source(String source, String jenkinsjob, String buildnumberorselector, String repo, String revision) { jenkinsjob(jenkinsjob, buildnumberorselector) releaserepo(repo, revision) setsource(source) From ecb225bff71552e2ac210a22a952b21f486837fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:35:26 +0000 Subject: [PATCH 15/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 75c4412..2ceb9f0 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -12,7 +12,7 @@ class ImportOptions { boolean appendtoparentmeta // Append to parent meta String source // Where are the artifacts coming from - String jenkinsjob + String jobname def buildnumberorselector String repo @@ -83,8 +83,8 @@ class ImportOptions { } @NonCPS - void jenkinsjob(String jenkinsjob, String buildnumberorselector) { - this.jenkinsjob = jenkinsjob.trim() + void jenkinsjob(String jobname, String buildnumberorselector) { + this.jobname = jobname.trim() this.buildnumberorselector = buildnumberorselector.trim() setsource('Jenkins job artifact') } @@ -97,8 +97,8 @@ class ImportOptions { } @NonCPS - void source(String source, String jenkinsjob, String buildnumberorselector, String repo, String revision) { - jenkinsjob(jenkinsjob, buildnumberorselector) + void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { + jenkinsjob(jobname, buildnumberorselector) releaserepo(repo, revision) setsource(source) } @@ -118,7 +118,7 @@ def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpat filter: importoptions.filter, excludes: importoptions.excludes, fingerprintArtifacts: fingerprint, - projectName: importoptions.jenkinsjob, + projectName: importoptions.jobname, selector: [ $class: 'ParameterizedBuildSelector', parameterName: importoptions.buildselector From b303e3f1455a37c85df20d5b4aa6c81338a8bab6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:37:03 +0000 Subject: [PATCH 16/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 2ceb9f0..2a6d095 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -28,7 +28,7 @@ class ImportOptions { appendtoparentmeta = true) { this.script = script setsource(source) - setIncrementVersionNumber(incrementversionnumber) + setincrementversionnumber(incrementversionnumber) setfilter(filter) setexcludes(excludes) setfingerprint(fingerprint) @@ -53,7 +53,7 @@ class ImportOptions { } @NonCPS - void setIncrementVersionNumber(boolean incrementversionnumber) { + void setincrementversionnumber(boolean incrementversionnumber) { this.incrementversionnumber = incrementversionnumber } From 26e471b6970dbffd032779b33d18cc4f2710482f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:38:33 +0000 Subject: [PATCH 17/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 2a6d095..d1dcd61 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -117,7 +117,7 @@ def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpat copyArtifacts( filter: importoptions.filter, excludes: importoptions.excludes, - fingerprintArtifacts: fingerprint, + fingerprintArtifacts: importoptions.fingerprint, projectName: importoptions.jobname, selector: [ $class: 'ParameterizedBuildSelector', From 44e69930c8602f109fa4f5cc04b9009cbee58930 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 16:40:02 +0000 Subject: [PATCH 18/54] GIS-1621 - Import for version numbering but don't append to meta data --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index d1dcd61..21c701e 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -121,7 +121,7 @@ def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpat projectName: importoptions.jobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: importoptions.buildselector + parameterName: importoptions.buildnumberorselector ], target: destpath ) From 7af31083e02b6f691d765555194a6270f069489a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 20 Dec 2019 17:31:14 +0000 Subject: [PATCH 19/54] GIS-1621 - Import for version numbering but don't append to meta data --- differences/dometahierarchy.groovy | 4 ++-- import/importer.groovy | 9 +++------ meta/folder.groovy | 6 ++++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 2e94820..3642ced 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -720,8 +720,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - def earlymeta = meta.importrelated(earlymetafolder) - def latermeta = meta.importrelated(latermetafolder) + def earlymeta = meta.importrelated(true, earlymetafolder) + def latermeta = meta.importrelated(true, latermetafolder) def earlymetaitems = earlymeta.releasewalkerunique(md) def latermetaitems = latermeta.releasewalkerunique(md) diff --git a/import/importer.groovy b/import/importer.groovy index 21c701e..8998ab5 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -82,21 +82,18 @@ class ImportOptions { this.appendtoparentmeta = appendtoparentmeta } - @NonCPS void jenkinsjob(String jobname, String buildnumberorselector) { this.jobname = jobname.trim() this.buildnumberorselector = buildnumberorselector.trim() setsource('Jenkins job artifact') } - @NonCPS void releaserepo(String repo, String revision) { this.repo = repo.trim() this.revision = revision.trim() setsource('Release repository') } - @NonCPS void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { jenkinsjob(jobname, buildnumberorselector) releaserepo(repo, revision) @@ -125,7 +122,7 @@ def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpat ], target: destpath ) - def importedmeta = meta.importrelated(destpath) + def importedmeta = meta.importrelated(importoptions.appendtoparentmeta, destpath) if (importoptions.incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } @@ -151,7 +148,7 @@ def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String de def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + def importedmeta = meta.importrelated(importoptions.appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) relatedimportinfo = importedmeta.getassociated() echo "Cloning associated repos" @@ -195,7 +192,7 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + def importedmeta = meta.importrelated(importoptions.appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" diff --git a/meta/folder.groovy b/meta/folder.groovy index 40f03aa..78286fe 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -605,7 +605,7 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME + "/" + METAFILENAME } - MetaInfo importrelated(String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { + MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { MetaInfo mi = new MetaInfo(script) def String adjustedpath = relativize(destpath) @@ -618,7 +618,9 @@ class MetaInfo extends GISJsonSerialised { } RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) - associatedmeta.add(rmi) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } return mi } From 0d53bf9f5852cf153caaea6d3efbf74873e62a0f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 21 Dec 2019 20:00:54 +0000 Subject: [PATCH 20/54] GIS-1621 - Use new form of importer method --- install/installhelper.groovy | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6fcdc6c..34a33d8 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -35,7 +35,16 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { for (artifactKey in artifactKeys) { (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) + + def importoptions = importer.getNewImportOptions(this) + + importoptions.jenkinsjob(sourceProject, sourceselector) + + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(toCopy) + importoptions.setexcludes(notToCopy) + + def importmeta = importer.artifactimport(meta, importoptions, destFolder) artifactFetched.put(artifactKey, importmeta) } } From 48dabad8f81b56e7ceb3a2f5cb75b802224f983f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 21 Dec 2019 20:09:02 +0000 Subject: [PATCH 21/54] GIS-1621 - Make the importoptions class CPS compliant by not calling methods from the constructor --- import/importer.groovy | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 8998ab5..898ee96 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -27,16 +27,15 @@ class ImportOptions { boolean nested = false, appendtoparentmeta = true) { this.script = script - setsource(source) - setincrementversionnumber(incrementversionnumber) - setfilter(filter) - setexcludes(excludes) - setfingerprint(fingerprint) - setnested(nested) - setappendtoparentmeta(appendtoparentmeta) + this.source = source.trim() + this.incrementversionnumber = incrementversionnumber + this.filter = filter.trim() + this.excludes = excludes.trim() + this.fingerprint = fingerprint + this.nested = nested + this.appendtoparentmeta = appendtoparentmeta } - @NonCPS void setsource(String source) { source = source.trim() switch (source) { @@ -52,32 +51,26 @@ class ImportOptions { } } - @NonCPS void setincrementversionnumber(boolean incrementversionnumber) { this.incrementversionnumber = incrementversionnumber } - @NonCPS void setfilter(String filter) { this.filter = filter.trim() } - @NonCPS void setexcludes(String excludes) { this.excludes = excludes.trim() } - @NonCPS void setfingerprint(boolean fingerprint) { this.fingerprint = fingerprint } - @NonCPS void setnested(boolean nested) { this.nested = nested } - @NonCPS void setappendtoparentmeta(boolean appendtoparentmeta) { this.appendtoparentmeta = appendtoparentmeta } @@ -234,7 +227,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false, boolean nested = false) { - importoptions = getNewImportOptions(this) + def importoptions = getNewImportOptions(this) importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) importoptions.setincrementversionnumber(incrementversionnumber) @@ -255,7 +248,7 @@ def artifactimport(meta, String filter = '**', String excludes = '', boolean fingerprint = false) { - importoptions = getNewImportOptions(this) + def importoptions = getNewImportOptions(this) importoptions.source(jenkinsjob, specific(buildnumber.toString())) importoptions.setincrementversionnumber(incrementversionnumber) From 390492bc53594836b70ef428f79969b6d92443a3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 21 Dec 2019 21:24:15 +0000 Subject: [PATCH 22/54] GIS-1621 - Import importer from a different scope --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 34a33d8..10c7827 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,3 +1,4 @@ +importer = load("${autobuilder.commonscripts()}/import/importer.groovy") import hudson.FilePath // Transfer file from intermediate position to final position; @@ -26,7 +27,6 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] - def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] if (artifactKeys instanceof java.lang.String) { From 7e72b0522faeac82d82243c9d0bb14e0ffb48b71 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 21 Dec 2019 21:33:11 +0000 Subject: [PATCH 23/54] GIS-1621 - Import importer from a different scope --- import/importer.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 898ee96..10d51b6 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -94,10 +94,6 @@ class ImportOptions { } } -ImportOptions getNewImportOptions(script) { - return new ImportOptions(script) -} - def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { if (importoptions.source != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") @@ -259,4 +255,8 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } +ImportOptions getNewImportOptions() { + return new ImportOptions(this) +} + return this; From f1c17a385f2c715772c11c350acd634dacf2f788 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 21 Dec 2019 21:40:29 +0000 Subject: [PATCH 24/54] GIS-1621 - Don't supply the script object to the importer --- import/importer.groovy | 4 ++-- install/installhelper.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 10d51b6..61f0daa 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -223,7 +223,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false, boolean nested = false) { - def importoptions = getNewImportOptions(this) + def importoptions = getNewImportOptions() importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) importoptions.setincrementversionnumber(incrementversionnumber) @@ -244,7 +244,7 @@ def artifactimport(meta, String filter = '**', String excludes = '', boolean fingerprint = false) { - def importoptions = getNewImportOptions(this) + def importoptions = getNewImportOptions() importoptions.source(jenkinsjob, specific(buildnumber.toString())) importoptions.setincrementversionnumber(incrementversionnumber) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 10c7827..e02a066 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -36,7 +36,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importoptions = importer.getNewImportOptions(this) + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(sourceProject, sourceselector) From 6621b2c402699136d90e993bcaa2a644755dc2c0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 09:56:05 +0000 Subject: [PATCH 25/54] GIS-1621 - Try to stop jenkins crash --- install/installhelper.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index e02a066..67578d7 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -27,6 +27,7 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] + def importoptions = importer.getNewImportOptions() for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] if (artifactKeys instanceof java.lang.String) { @@ -35,11 +36,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { for (artifactKey in artifactKeys) { (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - - def importoptions = importer.getNewImportOptions() - importoptions.jenkinsjob(sourceProject, sourceselector) - importoptions.setincrementversionnumber(incrementversionnumber) importoptions.setfilter(toCopy) importoptions.setexcludes(notToCopy) From c6ef2c58ad729202abd3a06f97a3a8f1fa26d496 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 10:02:20 +0000 Subject: [PATCH 26/54] GIS-1621 - Try to stop jenkins crash - member names --- import/importer.groovy | 130 ++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 65 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 61f0daa..3333cb4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -2,21 +2,21 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") class ImportOptions { - def script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - boolean incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported - String filter // Only import files/paths/folders that match the filter - String excludes // Exclude files/paths/folders that match the exclude filter - boolean fingerprint // Do Jenkins fingerprinting on imported artifacts - boolean nested // Import all child artifacts too (identified in the related meta info - boolean appendtoparentmeta // Append to parent meta - String source // Where are the artifacts coming from + boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String m_filter // Only import files/paths/folders that match the filter + String m_excludes // Exclude files/paths/folders that match the exclude filter + boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_appendtoparentmeta // Append to parent meta + String m_source // Where are the artifacts coming from - String jobname - def buildnumberorselector + String m_jobname + def m_buildnumberorselector - String repo - String revision + String m_repo + String m_revision ImportOptions(script, String source = 'Jenkins job artifact', @@ -26,25 +26,25 @@ class ImportOptions { boolean fingerprint = false, boolean nested = false, appendtoparentmeta = true) { - this.script = script - this.source = source.trim() - this.incrementversionnumber = incrementversionnumber - this.filter = filter.trim() - this.excludes = excludes.trim() - this.fingerprint = fingerprint - this.nested = nested - this.appendtoparentmeta = appendtoparentmeta + m_script = script + m_source = source.trim() + m_incrementversionnumber = incrementversionnumber + m_filter = filter.trim() + m_excludes = excludes.trim() + m_fingerprint = fingerprint + m_nested = nested + m_appendtoparentmeta = appendtoparentmeta } void setsource(String source) { source = source.trim() switch (source) { case 'Jenkins job artefact': - this.source = 'Jenkins job artifact' + m_source = 'Jenkins job artifact' break case 'Jenkins job artifact': case 'Release repository': - this.source = source + m_source = source break default: error("Unknown type of artifact source: ${source}") @@ -52,38 +52,38 @@ class ImportOptions { } void setincrementversionnumber(boolean incrementversionnumber) { - this.incrementversionnumber = incrementversionnumber + m_incrementversionnumber = incrementversionnumber } void setfilter(String filter) { - this.filter = filter.trim() + m_filter = filter.trim() } void setexcludes(String excludes) { - this.excludes = excludes.trim() + m_excludes = excludes.trim() } void setfingerprint(boolean fingerprint) { - this.fingerprint = fingerprint + m_fingerprint = fingerprint } void setnested(boolean nested) { - this.nested = nested + m_nested = nested } void setappendtoparentmeta(boolean appendtoparentmeta) { - this.appendtoparentmeta = appendtoparentmeta + m_appendtoparentmeta = appendtoparentmeta } void jenkinsjob(String jobname, String buildnumberorselector) { - this.jobname = jobname.trim() - this.buildnumberorselector = buildnumberorselector.trim() + m_jobname = jobname.trim() + m_buildnumberorselector = buildnumberorselector.trim() setsource('Jenkins job artifact') } void releaserepo(String repo, String revision) { - this.repo = repo.trim() - this.revision = revision.trim() + m_repo = repo.trim() + m_revision = revision.trim() setsource('Release repository') } @@ -95,24 +95,24 @@ class ImportOptions { } def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { - if (importoptions.source != 'Jenkins job artifact') { + if (importoptions.m_source != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") return } copyArtifacts( - filter: importoptions.filter, - excludes: importoptions.excludes, - fingerprintArtifacts: importoptions.fingerprint, - projectName: importoptions.jobname, + filter: importoptions.m_filter, + excludes: importoptions.m_excludes, + fingerprintArtifacts: importoptions.m_fingerprint, + projectName: importoptions.m_jobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: importoptions.buildnumberorselector + parameterName: importoptions.m_buildnumberorselector ], target: destpath ) - def importedmeta = meta.importrelated(importoptions.appendtoparentmeta, destpath) - if (importoptions.incrementversionnumber) { + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) + if (importoptions.m_incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta @@ -125,19 +125,19 @@ def rewritemetafileasreleased(meta, String filepath) { } def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { - if (importoptions.source != 'Release repository') { + if (importoptions.m_source != 'Release repository') { error("Import options settings have not been set for: Release repository") return } - def rooturl = importoptions.repo - def rootrevision = importoptions.revision + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(importoptions.appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) relatedimportinfo = importedmeta.getassociated() echo "Cloning associated repos" @@ -162,45 +162,45 @@ def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String de echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - if (importoptions.incrementversionnumber) { + if (importoptions.m_incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { - if (importoptions.source != 'Release repository') { + if (importoptions.m_source != 'Release repository') { error("Import options settings have not been set for: Release repository") return } - def rooturl = importoptions.repo - def rootrevision = importoptions.revision + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(importoptions.appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - if (importoptions.incrementversionnumber) { + if (importoptions.m_incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } def artifactimport(meta, ImportOptions importoptions, String destpath) { - switch (importoptions.source) { + switch (importoptions.m_source) { case 'Jenkins job artifact': case 'Jenkins job artefact': return artifactimportfrombuildjob(meta, importoptions, destpath) case 'Release repository': - if (importoptions.nested) { + if (importoptions.m_nested) { return hierarchyartifactimportfromrepo(meta, importoptions, destpath) } return artifactimportfromrepo(meta, importoptions, destpath) @@ -225,12 +225,12 @@ def artifactimport(meta, boolean nested = false) { def importoptions = getNewImportOptions() - importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - importoptions.setnested(nested) + importoptions.m_source(source, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.m_setincrementversionnumber(incrementversionnumber) + importoptions.m_setfilter(filter) + importoptions.m_setexcludes(excludes) + importoptions.m_setfingerprint(fingerprint) + importoptions.m_setnested(nested) return artifactimport(meta, importoptions, destpath) } @@ -246,11 +246,11 @@ def artifactimport(meta, boolean fingerprint = false) { def importoptions = getNewImportOptions() - importoptions.source(jenkinsjob, specific(buildnumber.toString())) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) + importoptions.m_source(jenkinsjob, specific(buildnumber.toString())) + importoptions.m_setincrementversionnumber(incrementversionnumber) + importoptions.m_setfilter(filter) + importoptions.m_setexcludes(excludes) + importoptions.m_setfingerprint(fingerprint) return artifactimport(meta, importoptions, destpath) } From faccc705336a5891d467cbffbb9766525a6942fc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 10:19:40 +0000 Subject: [PATCH 27/54] GIS-1621 - Try to stop jenkins crash - member names --- import/importer.groovy | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 3333cb4..0068257 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -225,12 +225,12 @@ def artifactimport(meta, boolean nested = false) { def importoptions = getNewImportOptions() - importoptions.m_source(source, jenkinsjob, buildnumberorselector, repo, revision) - importoptions.m_setincrementversionnumber(incrementversionnumber) - importoptions.m_setfilter(filter) - importoptions.m_setexcludes(excludes) - importoptions.m_setfingerprint(fingerprint) - importoptions.m_setnested(nested) + importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + importoptions.setnested(nested) return artifactimport(meta, importoptions, destpath) } @@ -246,11 +246,11 @@ def artifactimport(meta, boolean fingerprint = false) { def importoptions = getNewImportOptions() - importoptions.m_source(jenkinsjob, specific(buildnumber.toString())) - importoptions.m_setincrementversionnumber(incrementversionnumber) - importoptions.m_setfilter(filter) - importoptions.m_setexcludes(excludes) - importoptions.m_setfingerprint(fingerprint) + importoptions.source(jenkinsjob, specific(buildnumber.toString())) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) return artifactimport(meta, importoptions, destpath) } From d2a67a28dba92e567063ccf5a0da5be1e80bd970 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 17:47:36 +0000 Subject: [PATCH 28/54] GIS-1621 - Try to stop jenkins crash - move importer definition after update to JDK 11.0.5 indicated duplicate definition --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 67578d7..317faac 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,4 +1,3 @@ -importer = load("${autobuilder.commonscripts()}/import/importer.groovy") import hudson.FilePath // Transfer file from intermediate position to final position; @@ -27,6 +26,7 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def importoptions = importer.getNewImportOptions() for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] From 63be27007d20d30c030b8e0dce4bbf3f4931ddbd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 22:18:51 +0000 Subject: [PATCH 29/54] GIS-1621 - Try to stop jenkins crash - Seperate importoptions class into its own file --- import/importer.groovy | 96 +---------------------------------------- import/options.groovy | 98 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 94 deletions(-) create mode 100644 import/options.groovy diff --git a/import/importer.groovy b/import/importer.groovy index 0068257..4169671 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,98 +1,6 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - -class ImportOptions { - def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - - boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported - String m_filter // Only import files/paths/folders that match the filter - String m_excludes // Exclude files/paths/folders that match the exclude filter - boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts - boolean m_nested // Import all child artifacts too (identified in the related meta info - boolean m_appendtoparentmeta // Append to parent meta - String m_source // Where are the artifacts coming from - - String m_jobname - def m_buildnumberorselector - - String m_repo - String m_revision - - ImportOptions(script, - String source = 'Jenkins job artifact', - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false, - boolean nested = false, - appendtoparentmeta = true) { - m_script = script - m_source = source.trim() - m_incrementversionnumber = incrementversionnumber - m_filter = filter.trim() - m_excludes = excludes.trim() - m_fingerprint = fingerprint - m_nested = nested - m_appendtoparentmeta = appendtoparentmeta - } - - void setsource(String source) { - source = source.trim() - switch (source) { - case 'Jenkins job artefact': - m_source = 'Jenkins job artifact' - break - case 'Jenkins job artifact': - case 'Release repository': - m_source = source - break - default: - error("Unknown type of artifact source: ${source}") - } - } - - void setincrementversionnumber(boolean incrementversionnumber) { - m_incrementversionnumber = incrementversionnumber - } - - void setfilter(String filter) { - m_filter = filter.trim() - } - - void setexcludes(String excludes) { - m_excludes = excludes.trim() - } - - void setfingerprint(boolean fingerprint) { - m_fingerprint = fingerprint - } - - void setnested(boolean nested) { - m_nested = nested - } - - void setappendtoparentmeta(boolean appendtoparentmeta) { - m_appendtoparentmeta = appendtoparentmeta - } - - void jenkinsjob(String jobname, String buildnumberorselector) { - m_jobname = jobname.trim() - m_buildnumberorselector = buildnumberorselector.trim() - setsource('Jenkins job artifact') - } - - void releaserepo(String repo, String revision) { - m_repo = repo.trim() - m_revision = revision.trim() - setsource('Release repository') - } - - void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { - jenkinsjob(jobname, buildnumberorselector) - releaserepo(repo, revision) - setsource(source) - } -} +io = load("${autobuilder.commonscripts()}/import/options.groovy") def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_source != 'Jenkins job artifact') { @@ -256,7 +164,7 @@ def artifactimport(meta, } ImportOptions getNewImportOptions() { - return new ImportOptions(this) + return io.getNewImportOptions() } return this; diff --git a/import/options.groovy b/import/options.groovy new file mode 100644 index 0000000..8a478ec --- /dev/null +++ b/import/options.groovy @@ -0,0 +1,98 @@ +class ImportOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String m_filter // Only import files/paths/folders that match the filter + String m_excludes // Exclude files/paths/folders that match the exclude filter + boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_appendtoparentmeta // Append to parent meta + String m_source // Where are the artifacts coming from + + String m_jobname + def m_buildnumberorselector + + String m_repo + String m_revision + + ImportOptions(script, + String source = 'Jenkins job artifact', + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false, + appendtoparentmeta = true) { + m_script = script + m_source = source.trim() + m_incrementversionnumber = incrementversionnumber + m_filter = filter.trim() + m_excludes = excludes.trim() + m_fingerprint = fingerprint + m_nested = nested + m_appendtoparentmeta = appendtoparentmeta + } + + void setsource(String source) { + source = source.trim() + switch (source) { + case 'Jenkins job artefact': + m_source = 'Jenkins job artifact' + break + case 'Jenkins job artifact': + case 'Release repository': + m_source = source + break + default: + error("Unknown type of artifact source: ${source}") + } + } + + void setincrementversionnumber(boolean incrementversionnumber) { + m_incrementversionnumber = incrementversionnumber + } + + void setfilter(String filter) { + m_filter = filter.trim() + } + + void setexcludes(String excludes) { + m_excludes = excludes.trim() + } + + void setfingerprint(boolean fingerprint) { + m_fingerprint = fingerprint + } + + void setnested(boolean nested) { + m_nested = nested + } + + void setappendtoparentmeta(boolean appendtoparentmeta) { + m_appendtoparentmeta = appendtoparentmeta + } + + void jenkinsjob(String jobname, String buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = buildnumberorselector.trim() + setsource('Jenkins job artifact') + } + + void releaserepo(String repo, String revision) { + m_repo = repo.trim() + m_revision = revision.trim() + setsource('Release repository') + } + + void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { + jenkinsjob(jobname, buildnumberorselector) + releaserepo(repo, revision) + setsource(source) + } +} + +ImportOptions getNewImportOptions() { + return new ImportOptions(this) +} + +return this; \ No newline at end of file From ab0d0d5ae08d3ddb20b9432762ba1413e754895d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 22:24:27 +0000 Subject: [PATCH 30/54] GIS-1621 - Try to stop jenkins crash - Seperate importoptions class into its own file --- import/importer.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 4169671..1824c93 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -32,7 +32,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { +def hierarchyartifactimportfromrepo(meta, io.ImportOptions importoptions, String destpath) { if (importoptions.m_source != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -76,7 +76,7 @@ def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String de return importedmeta } -def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { +def artifactimportfromrepo(meta, io.ImportOptions importoptions, String destpath) { if (importoptions.m_source != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -102,7 +102,7 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { return importedmeta } -def artifactimport(meta, ImportOptions importoptions, String destpath) { +def artifactimport(meta, io.ImportOptions importoptions, String destpath) { switch (importoptions.m_source) { case 'Jenkins job artifact': case 'Jenkins job artefact': From 5a3483b950c338194ade5d9965a28185c33f3e85 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 22:31:31 +0000 Subject: [PATCH 31/54] GIS-1621 - Try to stop jenkins crash - Seperate importoptions class into its own file --- import/importer.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 1824c93..ef15c91 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -2,7 +2,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") io = load("${autobuilder.commonscripts()}/import/options.groovy") -def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { +def artifactimportfrombuildjob(meta, importoptions, String destpath) { if (importoptions.m_source != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") return @@ -32,7 +32,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, io.ImportOptions importoptions, String destpath) { +def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { if (importoptions.m_source != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -76,7 +76,7 @@ def hierarchyartifactimportfromrepo(meta, io.ImportOptions importoptions, String return importedmeta } -def artifactimportfromrepo(meta, io.ImportOptions importoptions, String destpath) { +def artifactimportfromrepo(meta, importoptions, String destpath) { if (importoptions.m_source != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -102,7 +102,7 @@ def artifactimportfromrepo(meta, io.ImportOptions importoptions, String destpath return importedmeta } -def artifactimport(meta, io.ImportOptions importoptions, String destpath) { +def artifactimport(meta, importoptions, String destpath) { switch (importoptions.m_source) { case 'Jenkins job artifact': case 'Jenkins job artefact': @@ -163,7 +163,7 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } -ImportOptions getNewImportOptions() { +def getNewImportOptions() { return io.getNewImportOptions() } From be5bb5fa77f9e62f52ab743bf0276b33236e5acd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 22:47:06 +0000 Subject: [PATCH 32/54] GIS-1621 - Try to stop jenkins crash - Attempt to eliminate double load of importer --- install/installhelper.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 317faac..96a8b2e 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -23,10 +23,9 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { artifactFetched = [:] - def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def importoptions = importer.getNewImportOptions() for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] From aaa96f26d9d685c61e5a9f598e274ddac9e10e5b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Dec 2019 22:56:33 +0000 Subject: [PATCH 33/54] GIS-1621 - Try to stop jenkins crash - Move importer options class back into importer --- import/importer.groovy | 102 +++++++++++++++++++++++++++++++++++++++-- import/options.groovy | 98 --------------------------------------- 2 files changed, 97 insertions(+), 103 deletions(-) delete mode 100644 import/options.groovy diff --git a/import/importer.groovy b/import/importer.groovy index ef15c91..784e1f5 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,6 +1,102 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -io = load("${autobuilder.commonscripts()}/import/options.groovy") + +class ImportOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String m_filter // Only import files/paths/folders that match the filter + String m_excludes // Exclude files/paths/folders that match the exclude filter + boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_appendtoparentmeta // Append to parent meta + String m_source // Where are the artifacts coming from + + String m_jobname + def m_buildnumberorselector + + String m_repo + String m_revision + + ImportOptions(script, + String source = 'Jenkins job artifact', + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false, + appendtoparentmeta = true) { + m_script = script + m_source = source.trim() + m_incrementversionnumber = incrementversionnumber + m_filter = filter.trim() + m_excludes = excludes.trim() + m_fingerprint = fingerprint + m_nested = nested + m_appendtoparentmeta = appendtoparentmeta + } + + void setsource(String source) { + source = source.trim() + switch (source) { + case 'Jenkins job artefact': + m_source = 'Jenkins job artifact' + break + case 'Jenkins job artifact': + case 'Release repository': + m_source = source + break + default: + error("Unknown type of artifact source: ${source}") + } + } + + void setincrementversionnumber(boolean incrementversionnumber) { + m_incrementversionnumber = incrementversionnumber + } + + void setfilter(String filter) { + m_filter = filter.trim() + } + + void setexcludes(String excludes) { + m_excludes = excludes.trim() + } + + void setfingerprint(boolean fingerprint) { + m_fingerprint = fingerprint + } + + void setnested(boolean nested) { + m_nested = nested + } + + void setappendtoparentmeta(boolean appendtoparentmeta) { + m_appendtoparentmeta = appendtoparentmeta + } + + void jenkinsjob(String jobname, String buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = buildnumberorselector.trim() + setsource('Jenkins job artifact') + } + + void releaserepo(String repo, String revision) { + m_repo = repo.trim() + m_revision = revision.trim() + setsource('Release repository') + } + + void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { + jenkinsjob(jobname, buildnumberorselector) + releaserepo(repo, revision) + setsource(source) + } +} + +ImportOptions getNewImportOptions() { + return new ImportOptions(this) +} def artifactimportfrombuildjob(meta, importoptions, String destpath) { if (importoptions.m_source != 'Jenkins job artifact') { @@ -163,8 +259,4 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } -def getNewImportOptions() { - return io.getNewImportOptions() -} - return this; diff --git a/import/options.groovy b/import/options.groovy deleted file mode 100644 index 8a478ec..0000000 --- a/import/options.groovy +++ /dev/null @@ -1,98 +0,0 @@ -class ImportOptions { - def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - - boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported - String m_filter // Only import files/paths/folders that match the filter - String m_excludes // Exclude files/paths/folders that match the exclude filter - boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts - boolean m_nested // Import all child artifacts too (identified in the related meta info - boolean m_appendtoparentmeta // Append to parent meta - String m_source // Where are the artifacts coming from - - String m_jobname - def m_buildnumberorselector - - String m_repo - String m_revision - - ImportOptions(script, - String source = 'Jenkins job artifact', - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false, - boolean nested = false, - appendtoparentmeta = true) { - m_script = script - m_source = source.trim() - m_incrementversionnumber = incrementversionnumber - m_filter = filter.trim() - m_excludes = excludes.trim() - m_fingerprint = fingerprint - m_nested = nested - m_appendtoparentmeta = appendtoparentmeta - } - - void setsource(String source) { - source = source.trim() - switch (source) { - case 'Jenkins job artefact': - m_source = 'Jenkins job artifact' - break - case 'Jenkins job artifact': - case 'Release repository': - m_source = source - break - default: - error("Unknown type of artifact source: ${source}") - } - } - - void setincrementversionnumber(boolean incrementversionnumber) { - m_incrementversionnumber = incrementversionnumber - } - - void setfilter(String filter) { - m_filter = filter.trim() - } - - void setexcludes(String excludes) { - m_excludes = excludes.trim() - } - - void setfingerprint(boolean fingerprint) { - m_fingerprint = fingerprint - } - - void setnested(boolean nested) { - m_nested = nested - } - - void setappendtoparentmeta(boolean appendtoparentmeta) { - m_appendtoparentmeta = appendtoparentmeta - } - - void jenkinsjob(String jobname, String buildnumberorselector) { - m_jobname = jobname.trim() - m_buildnumberorselector = buildnumberorselector.trim() - setsource('Jenkins job artifact') - } - - void releaserepo(String repo, String revision) { - m_repo = repo.trim() - m_revision = revision.trim() - setsource('Release repository') - } - - void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { - jenkinsjob(jobname, buildnumberorselector) - releaserepo(repo, revision) - setsource(source) - } -} - -ImportOptions getNewImportOptions() { - return new ImportOptions(this) -} - -return this; \ No newline at end of file From 697afbb7e31005f0c0742b10e1c41a19cfa9ed6d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 13:14:36 +0000 Subject: [PATCH 34/54] GIS-1621 - Create another deprecated overload of artifactimport --- import/importer.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/import/importer.groovy b/import/importer.groovy index 784e1f5..bdb7faa 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -259,4 +259,24 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + buildnumberorselector, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false) { + def importoptions = getNewImportOptions() + + importoptions.jenkinsjob(buildjob, buildnumberorselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + return this; From 106f6b92a1981ae55c0cfbfc34e14c66d91f6dbf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 13:15:11 +0000 Subject: [PATCH 35/54] GIS-1621 - Remove redundant declaration --- differences/dometahierarchy.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3642ced..199bd2c 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -140,7 +140,6 @@ def String getcomponent(meta) { } def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { - def differencerecords = [] def String releaseexpression if (tripledot) { releaseexpression = earlyrevision + "..." + laterrevision From 8aa50af0558c10e51904ba59fc253c7c7493b69f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 13:20:09 +0000 Subject: [PATCH 36/54] GIS-1621 - Echo deprecated methods --- import/importer.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/import/importer.groovy b/import/importer.groovy index bdb7faa..418079e 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -227,6 +227,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false, boolean nested = false) { + echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" def importoptions = getNewImportOptions() importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) @@ -248,6 +249,7 @@ def artifactimport(meta, String filter = '**', String excludes = '', boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, ...)" def importoptions = getNewImportOptions() importoptions.source(jenkinsjob, specific(buildnumber.toString())) @@ -268,6 +270,7 @@ def artifactimport(meta, String filter = '', String excludes = '', boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" def importoptions = getNewImportOptions() importoptions.jenkinsjob(buildjob, buildnumberorselector) From efaa302b544bfbf8c545d1803e20b5f4b863c490 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 14:31:18 +0000 Subject: [PATCH 37/54] GIS-1621 - Extract differences from squashed commits --- differences/dometahierarchy.groovy | 84 +++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 19 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 199bd2c..d484949 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -166,6 +166,16 @@ def extractsha(String line) { return [sha.trim(), restofline.trim()] } +def extractsquashes(String line) { + def pattern1 = "^(.*)(Squashed .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + return [false, line] +} + def extractmerges(String line) { def pattern1 = "^(.*)(Merge branch .*)" def match1 = (line =~ pattern1) @@ -302,7 +312,35 @@ def extractreverts(String line) { return [false, [], line] } -def DifferenceRecord fillrecord(meta, line) { +def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() + } + return differencetext +} + +def List fillsquashrecords(String sourcepath, String sha) { + def List differencerecords = [] + + String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) + int count = 0 + squashloglines.split('\n').each { + if (count > 0) { // Ignore first line + def line = it + if (line?.trim()) { + def DifferenceRecord dr = fillrecord(meta, line.trim(), sourcepath) + differencerecords.add(dr) + //echo "${dr.dump()}" + } + } + count += 1 + } + return differencerecords +} + +def List fillrecord(meta, line, String sourcepath) { + def List differencerecords = [] def DifferenceRecord dr = new DifferenceRecord() dr.productname = meta.productinfo.productname @@ -316,41 +354,49 @@ def DifferenceRecord fillrecord(meta, line) { (sha, line2) = extractsha(line) dr.sha = sha - - (result, mergeinfo) = extractmerges(line2) + + (result, squashinfo) = extractsquashes(line2) if (result) { - dr.type = "Merge" - dr.mergeinfo = mergeinfo + dr.type = "Squash" + dr.squashinfo = squashinfo + squashrecords = fillsquashrecords(sourcepath, sha) + differencerecords.addAll(squashrecords) } else { - (result, jiraticketinfo, comment) = extractreverts(line2) + (result, mergeinfo) = extractmerges(line2) if (result) { - dr.type = "Revert" - dr.jiraticketinfo = jiraticketinfo - dr.comment = comment + dr.type = "Merge" + dr.mergeinfo = mergeinfo } else { - (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + (result, jiraticketinfo, comment) = extractreverts(line2) if (result) { - dr.type = "Ticketed" + dr.type = "Revert" dr.jiraticketinfo = jiraticketinfo dr.comment = comment } else { - dr.type = "Unticketed" - dr.comment = line2 + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.comment = line2 + } } } } - - return dr + differencerecords.add(dr) + return differencerecords } -def List processtext(meta, String text) { +def List processtext(meta, String text, String sourcepath) { def List differencerecords = [] text.split('\n').each { def line = it if (line?.trim()) { - def DifferenceRecord dr = fillrecord(meta, line.trim()) - differencerecords.add(dr) + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) //echo "${dr.dump()}" } } @@ -370,7 +416,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - differencerecords = processtext(earlymeta, differencetext) + differencerecords = processtext(earlymeta, differencetext, sourcepath) if (earlyrevision == laterrevision) { summaryappend(" No change") } From 12c938887f6eed209f165004254611b8b588f623 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 15:17:57 +0000 Subject: [PATCH 38/54] GIS-1621 - Add squash info --- differences/dometahierarchy.groovy | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index d484949..eb7d6bb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -35,6 +35,7 @@ class DifferenceRecord { String sha String type String mergeinfo + String squashinfo List jiraticketinfo String comment @@ -47,6 +48,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" + squashinfo = "" jiraticketinfo = [] comment = "" } @@ -440,6 +442,28 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } +class SquashReportRecord { + String comment + int duplicates + + SquashReportRecord() { + comment = '' + duplicates = 0 + } +} + +def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { + dir(reportsubfolder) { + def String squashlist = "${title}\nSquash list:\n" + def squashessorted = squashes.sort() + for (asquash in squashessorted) { + def SquashReportRecord s = asquash.value + squashlist += "${m.comment} (${m.duplicates})\n" + } + writeFile(file: "mergelist.txt", text: squashlist) + } +} + class MergeReportRecord { String comment int duplicates @@ -592,12 +616,25 @@ def differencerecordsreport(String reportsubfolder, String title, differencereco def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] + def Map squashes = [:] def Map merges = [:] def Map reverts = [:] def Map unticketed = [:] def Map tickets = [:] for (dr in differencerecords) { + if (dr.type == "Squash") { + def SquashReportRecord s + if (!squashes.containsKey(dr.mergeinfo)) { + s = new SquashReportRecord() + s.comment = dr.squashinfo + s.duplicates = 1 + } else { + s = squashes[dr.squashinfo] + s.duplicates += 1 + } + squashes[dr.squashinfo] = s + } if (dr.type == "Merge") { def MergeReportRecord m if (!merges.containsKey(dr.mergeinfo)) { @@ -710,6 +747,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) { reportsubfolder = "${reportfolder}/Combined" } mergereport(reportsubfolder, title, merges, iscomponent) + squashreport(reportsubfolder, title, squashes, iscomponent) revertsreport(reportsubfolder, title, reverts, iscomponent) ticketedreport(reportsubfolder, title, ticketed, iscomponent) unticketedreport(reportsubfolder, title, unticketed, iscomponent) From 1f65877a5bb05593f3fc376495e4ae542660fb0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 15:22:12 +0000 Subject: [PATCH 39/54] GIS-1621 - Add squash info --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index eb7d6bb..16c78f3 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -331,8 +331,8 @@ def List fillsquashrecords(String sourcepath, String sha) { if (count > 0) { // Ignore first line def line = it if (line?.trim()) { - def DifferenceRecord dr = fillrecord(meta, line.trim(), sourcepath) - differencerecords.add(dr) + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) //echo "${dr.dump()}" } } From 5c27f561b856eb655ea769496f365db6734a9737 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 15:27:29 +0000 Subject: [PATCH 40/54] GIS-1621 - Add squash info --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 16c78f3..05f08de 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -458,7 +458,7 @@ def squashreport(String reportsubfolder, String title, LinkedHashMap Date: Mon, 23 Dec 2019 16:03:00 +0000 Subject: [PATCH 41/54] GIS-1621 - Add squash info --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 05f08de..2475f36 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -460,7 +460,7 @@ def squashreport(String reportsubfolder, String title, LinkedHashMap Date: Mon, 23 Dec 2019 16:32:33 +0000 Subject: [PATCH 42/54] GIS-1621 - metaexport must not import to importer --- artifacts/metaexport.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 043c4de..67d13c3 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,8 +1,6 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(meta, subJobs) { - def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - +def combineSubjobArtifacts(meta, importer, subJobs) { def ArrayList artifactFolders = new ArrayList() for (subJob in subJobs) { def subJobObject = subJob[0] From 39516d4c4ba2fbc6f012f48b4dce443d775858c2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 17:12:07 +0000 Subject: [PATCH 43/54] GIS-1621 - Make source method unique --- import/importer.groovy | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 418079e..7bf9b7d 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -10,7 +10,7 @@ class ImportOptions { boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts boolean m_nested // Import all child artifacts too (identified in the related meta info boolean m_appendtoparentmeta // Append to parent meta - String m_source // Where are the artifacts coming from + String m_sourceselect // Where are the artifacts coming from String m_jobname def m_buildnumberorselector @@ -19,7 +19,7 @@ class ImportOptions { String m_revision ImportOptions(script, - String source = 'Jenkins job artifact', + String sourceselect = 'Jenkins job artifact', boolean incrementversionnumber = true, String filter = '', String excludes = '', @@ -27,7 +27,7 @@ class ImportOptions { boolean nested = false, appendtoparentmeta = true) { m_script = script - m_source = source.trim() + m_sourceselect = sourceselect.trim() m_incrementversionnumber = incrementversionnumber m_filter = filter.trim() m_excludes = excludes.trim() @@ -36,18 +36,18 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void setsource(String source) { - source = source.trim() - switch (source) { + void setsourceselect(String sourceselect) { + sourceselect = sourceselect.trim() + switch (sourceselect) { case 'Jenkins job artefact': - m_source = 'Jenkins job artifact' + m_sourceselect = 'Jenkins job artifact' break case 'Jenkins job artifact': case 'Release repository': - m_source = source + m_sourceselect = sourceselect break default: - error("Unknown type of artifact source: ${source}") + error("Unknown type of artifact source select: ${m_sourceselect}") } } @@ -78,19 +78,19 @@ class ImportOptions { void jenkinsjob(String jobname, String buildnumberorselector) { m_jobname = jobname.trim() m_buildnumberorselector = buildnumberorselector.trim() - setsource('Jenkins job artifact') + setsourceselect('Jenkins job artifact') } void releaserepo(String repo, String revision) { m_repo = repo.trim() m_revision = revision.trim() - setsource('Release repository') + setsourceselect('Release repository') } - void source(String source, String jobname, String buildnumberorselector, String repo, String revision) { + void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) { jenkinsjob(jobname, buildnumberorselector) releaserepo(repo, revision) - setsource(source) + setsourceselect(sourceselect) } } @@ -99,7 +99,7 @@ ImportOptions getNewImportOptions() { } def artifactimportfrombuildjob(meta, importoptions, String destpath) { - if (importoptions.m_source != 'Jenkins job artifact') { + if (importoptions.m_sourceselect != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") return } @@ -129,7 +129,7 @@ def rewritemetafileasreleased(meta, String filepath) { } def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { - if (importoptions.m_source != 'Release repository') { + if (importoptions.m_sourceselect != 'Release repository') { error("Import options settings have not been set for: Release repository") return } @@ -173,7 +173,7 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { } def artifactimportfromrepo(meta, importoptions, String destpath) { - if (importoptions.m_source != 'Release repository') { + if (importoptions.m_sourceselect != 'Release repository') { error("Import options settings have not been set for: Release repository") return } @@ -199,7 +199,7 @@ def artifactimportfromrepo(meta, importoptions, String destpath) { } def artifactimport(meta, importoptions, String destpath) { - switch (importoptions.m_source) { + switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': return artifactimportfrombuildjob(meta, importoptions, destpath) @@ -216,7 +216,7 @@ def artifactimport(meta, importoptions, String destpath) { // Deprecated (use above) def artifactimport(meta, - String source, + String sourceselect, String jenkinsjob, buildnumberorselector, String repo, @@ -230,7 +230,7 @@ def artifactimport(meta, echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" def importoptions = getNewImportOptions() - importoptions.source(source, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) importoptions.setincrementversionnumber(incrementversionnumber) importoptions.setfilter(filter) importoptions.setexcludes(excludes) From 5319643bf47d2d34e943331e7a46553a0c697031 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 17:53:23 +0000 Subject: [PATCH 44/54] GIS-1621 - Correct deprecated method --- import/importer.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 7bf9b7d..87dca7c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -242,17 +242,17 @@ def artifactimport(meta, // Deprecated (use above) def artifactimport(meta, - String jenkinsjob, + String buildjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { - echo "DEPRECATED method: artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, ...)" + echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" def importoptions = getNewImportOptions() - importoptions.source(jenkinsjob, specific(buildnumber.toString())) + importoptions.jenkinsjob(buildjob, specific(buildnumber.toString())) importoptions.setincrementversionnumber(incrementversionnumber) importoptions.setfilter(filter) importoptions.setexcludes(excludes) From bdfa7aecbcc0d578c2a87b62bb99c3cd02452a9d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 21:04:00 +0000 Subject: [PATCH 45/54] GIS-1621 - Correct deprecated method --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 87dca7c..41f5118 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -252,7 +252,7 @@ def artifactimport(meta, echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" def importoptions = getNewImportOptions() - importoptions.jenkinsjob(buildjob, specific(buildnumber.toString())) + importoptions.jenkinsjob(buildjob, " ${buildnumber}") importoptions.setincrementversionnumber(incrementversionnumber) importoptions.setfilter(filter) importoptions.setexcludes(excludes) From e2dac1558f2640ed4c57073f2ce33287a65e77ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 23:32:23 +0000 Subject: [PATCH 46/54] GIS-1621 - Remove redundant parameters; Deprecate old APIs --- autobuilder.groovy | 19 +++++++++++-------- meta/folder.groovy | 9 ++------- release/registerrepo.groovy | 2 +- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 63ca2de..35b9f6f 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,10 +1,8 @@ // Assume that WORKSPACE environment variable is set -autobuildroot = "${WORKSPACE}/autobuild" - -commonscriptsroot = "${autobuildroot}/Common_scripts" - -metafolderscript = null +String autobuildroot = "${WORKSPACE}/autobuild" +String commonscriptsroot = "${autobuildroot}/Common_scripts" +def metafolderscript = null def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { @@ -32,7 +30,7 @@ def getmetafolderscript() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -42,13 +40,13 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set(productname, subcomponentname, jirakey) + meta.productinfo.set(productname, subcomponentname) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def rr = load("${commonscripts()}/release/registerrepo.groovy") - rr.registerrepo(meta, releaserepo, repoRootSuffix) + rr.registerrepo(meta, releaserepo) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) @@ -56,6 +54,11 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA return meta } +def start(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String jirakey = JIRA_KEY, String releaserepo = RELEASE_REPO, String repoRootSuffix = "", String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { + echo "DEPRECATED method: start(script, String productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, ...)" + return startbuild(script, productname, subcomponentname, releaserepo, autobuildpath, commonscriptspath) +} + // Start up boilderplate def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting non-build job...." diff --git a/meta/folder.groovy b/meta/folder.groovy index 78286fe..96c9626 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -22,16 +22,14 @@ abstract class GISJsonSerialised implements Serializable { class ProductInfo extends GISJsonSerialised { String productname String subcomponentname - String jirakey - ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + ProductInfo(script, productname = "", subcomponentname = "") { super(script) this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jirakey } - void set(String productname, String subcomponentname, String jirakey) { + void set(String productname, String subcomponentname) { // Make sure the fields can be used in git tags def int pstatus = 0 if (script.isUnix()) { @@ -55,20 +53,17 @@ class ProductInfo extends GISJsonSerialised { } this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jirakey } void deserialiseJson(rawmeta) { productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] - jirakey = rawmeta['jirakey'] } LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, - 'jirakey' : jirakey, ] return data } diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index bd6a528..f565706 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -2,7 +2,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { +def registerrepo(meta, String releaserepo) { if (releaserepo?.trim()) { vc.checkRepoExists(releaserepo) } From c5b293d4b61808ad615ac69c8f17dd2c06732f77 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 23 Dec 2019 23:41:36 +0000 Subject: [PATCH 47/54] GIS-1621 - Remove redundant parameters; Deprecate old APIs --- autobuilder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 35b9f6f..25fe486 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,8 +1,8 @@ // Assume that WORKSPACE environment variable is set -String autobuildroot = "${WORKSPACE}/autobuild" -String commonscriptsroot = "${autobuildroot}/Common_scripts" -def metafolderscript = null +autobuildroot = "${WORKSPACE}/autobuild" +commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 13a6534fac7267ab8196e0613827882f5d8db143 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 24 Dec 2019 08:09:30 +0000 Subject: [PATCH 48/54] GIS-1621 - Remove redundant parameters; Deprecate old APIs --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 25fe486..1256394 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -70,7 +70,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") + meta.productinfo.set("Non-build-product", "Non-build-subcomponent") def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) From 4ee6c594506da1bd0ea6e1d3958b0baeaa653635 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 29 Dec 2019 12:42:00 +0000 Subject: [PATCH 49/54] GIS-1621 - Remove further references to jirakey field; Remove redundant script --- differences/dometahierarchy.groovy | 5 +- differences/show_release_differences.py | 225 ------------------------ 2 files changed, 1 insertion(+), 229 deletions(-) delete mode 100644 differences/show_release_differences.py diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 2475f36..6280960 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -29,7 +29,6 @@ class JiraTicketInfo { class DifferenceRecord { String productname String subcomponentname - String jirakey String rawdifferencetext String sha @@ -42,7 +41,6 @@ class DifferenceRecord { DifferenceRecord() { productname = "" subcomponentname = "" - jirakey = "" rawdifferencetext = "" sha = "" @@ -58,7 +56,7 @@ class DifferenceRecord { for (j in jiraticketinfo) { jiraticketstring += j.dump() + " " } - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -347,7 +345,6 @@ def List fillrecord(meta, line, String sourcepath) { dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname - dr.jirakey = meta.productinfo.jirakey dr.rawdifferencetext = line diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py deleted file mode 100644 index 5c0f15d..0000000 --- a/differences/show_release_differences.py +++ /dev/null @@ -1,225 +0,0 @@ -''' -showreleasedifferences -- shortdesc - -showreleasedifferences generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import subprocess -import os -import sys -import re - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") - -class GitException(Exception): - def __init__(self, message): - self.message = message - - def __repr__(self): - return self.message - -def printDivider(outfh, times = 1, length = 100): - for _ in range(times): - outfh.write("{0}\n".format("*" * length)) - -TABSIZE = 4 -def printTabbed(outfh, string, tabs = 0): - outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) - -def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): - if tripledot: - releaseExpression = previousRelease + "..." + currentRelease - else: - releaseExpression = previousRelease + ".." + currentRelease - currentdir = os.getcwd() - os.chdir(gitrootfolder) - gitlog = [] - printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) - proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) - while True: - line = proc.stdout.readline() - if line: - linestring = line.decode('utf-8').rstrip() - gitlog.append(linestring) - #printTabbed(outfh, linestring, 1) - else: - break - proc.wait() - if proc.returncode > 0: - raise GitException("Git executable returned an error") - os.chdir(currentdir) - return gitlog - -jiraprojectpattern = { - 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), - ), - 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), - ), - 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), - ), - 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), - ), - 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), - ), - 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), - ), - 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), - ), - } - -mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") - -def processLog(gitlog, jirakeyprefix): - reverts = [] - merges = {} - other = [] - numbered = {} - (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] - for line in gitlog: - expectedmatch = expectedpattern.match(line) - revertmatch = revertpattern.match(line) - mergematch = mergepattern.match(line) - if expectedmatch == None and revertmatch == None and mergematch == None: - other.append(line) - if revertmatch: - gisnumber = jirakeyprefix + "-" + revertmatch.group(2) - description = revertmatch.group(3) - reverts.append("Revert: {0} {1}".format(gisnumber, description)) - if mergematch: - merge = mergematch.group(2) - if not merge in merges: - merges[merge] = 1 - else: - merges[merge] += 1 - if expectedmatch: - gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) - description = expectedmatch.group(2) - if not gisnumber in numbered: - numbered[gisnumber] = [] - numbered[gisnumber].append(description) - return (reverts, merges, other, numbered) - -def showDifferences(differences, jirakeyprefix, outfh): - (revertslog, mergeslog, otherlog, numberedlog) = differences - printDivider(outfh) - printTabbed(outfh, jirakeyprefix + "-Numbers Log:") - for (gisnumber, descriptions) in sorted(numberedlog.items()): - printTabbed(outfh, gisnumber, 1) - descriptions = numberedlog[gisnumber]; - for description in descriptions: - printTabbed(outfh, description, 2) - printDivider(outfh) - printTabbed(outfh, "Revert log:") - for reversion in sorted(revertslog): - printTabbed(outfh, reversion, 1) - printDivider(outfh) - printTabbed(outfh, "Merges log:") - for merge in sorted(mergeslog): - printTabbed(outfh, merge, 1) - printDivider(outfh) - printTabbed(outfh, "Other log:") - for other in sorted(otherlog): - printTabbed(outfh, other, 1) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") - parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") - parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") - parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - gitrootfolder = args.gitrootfolder - sourceversion = args.sourceversion - previousversion = args.previousversion - jirakeyprefix = args.jirakeyprefix - outputoptions = args.outputoptions - difffile = args.difffile - - print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - - if difffile: - os.makedirs(os.path.dirname(difffile), exist_ok=True) - outfh = open(difffile, 'w') - else: - outfh = sys.stdout - - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if 'opposite' in outputoptions.lower(): - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if 'notcommon' in outputoptions.lower(): - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - - if difffile: - outfh.close() - return 0 - except Exception as e: - print("Error while determining differences:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) From 4a9546d7d4710d76bc672883669279a42940dc5e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 Jan 2020 09:11:38 +0000 Subject: [PATCH 50/54] GIS-1621 - Bring EMB build into line with SM --- differences/show_release_differences.py | 225 ------------------------ 1 file changed, 225 deletions(-) delete mode 100644 differences/show_release_differences.py diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py deleted file mode 100644 index 5c0f15d..0000000 --- a/differences/show_release_differences.py +++ /dev/null @@ -1,225 +0,0 @@ -''' -showreleasedifferences -- shortdesc - -showreleasedifferences generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import subprocess -import os -import sys -import re - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") - -class GitException(Exception): - def __init__(self, message): - self.message = message - - def __repr__(self): - return self.message - -def printDivider(outfh, times = 1, length = 100): - for _ in range(times): - outfh.write("{0}\n".format("*" * length)) - -TABSIZE = 4 -def printTabbed(outfh, string, tabs = 0): - outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) - -def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): - if tripledot: - releaseExpression = previousRelease + "..." + currentRelease - else: - releaseExpression = previousRelease + ".." + currentRelease - currentdir = os.getcwd() - os.chdir(gitrootfolder) - gitlog = [] - printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) - proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) - while True: - line = proc.stdout.readline() - if line: - linestring = line.decode('utf-8').rstrip() - gitlog.append(linestring) - #printTabbed(outfh, linestring, 1) - else: - break - proc.wait() - if proc.returncode > 0: - raise GitException("Git executable returned an error") - os.chdir(currentdir) - return gitlog - -jiraprojectpattern = { - 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), - ), - 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), - ), - 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), - ), - 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), - ), - 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), - ), - 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), - ), - 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), - ), - } - -mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") - -def processLog(gitlog, jirakeyprefix): - reverts = [] - merges = {} - other = [] - numbered = {} - (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] - for line in gitlog: - expectedmatch = expectedpattern.match(line) - revertmatch = revertpattern.match(line) - mergematch = mergepattern.match(line) - if expectedmatch == None and revertmatch == None and mergematch == None: - other.append(line) - if revertmatch: - gisnumber = jirakeyprefix + "-" + revertmatch.group(2) - description = revertmatch.group(3) - reverts.append("Revert: {0} {1}".format(gisnumber, description)) - if mergematch: - merge = mergematch.group(2) - if not merge in merges: - merges[merge] = 1 - else: - merges[merge] += 1 - if expectedmatch: - gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) - description = expectedmatch.group(2) - if not gisnumber in numbered: - numbered[gisnumber] = [] - numbered[gisnumber].append(description) - return (reverts, merges, other, numbered) - -def showDifferences(differences, jirakeyprefix, outfh): - (revertslog, mergeslog, otherlog, numberedlog) = differences - printDivider(outfh) - printTabbed(outfh, jirakeyprefix + "-Numbers Log:") - for (gisnumber, descriptions) in sorted(numberedlog.items()): - printTabbed(outfh, gisnumber, 1) - descriptions = numberedlog[gisnumber]; - for description in descriptions: - printTabbed(outfh, description, 2) - printDivider(outfh) - printTabbed(outfh, "Revert log:") - for reversion in sorted(revertslog): - printTabbed(outfh, reversion, 1) - printDivider(outfh) - printTabbed(outfh, "Merges log:") - for merge in sorted(mergeslog): - printTabbed(outfh, merge, 1) - printDivider(outfh) - printTabbed(outfh, "Other log:") - for other in sorted(otherlog): - printTabbed(outfh, other, 1) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") - parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") - parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") - parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - gitrootfolder = args.gitrootfolder - sourceversion = args.sourceversion - previousversion = args.previousversion - jirakeyprefix = args.jirakeyprefix - outputoptions = args.outputoptions - difffile = args.difffile - - print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - - if difffile: - os.makedirs(os.path.dirname(difffile), exist_ok=True) - outfh = open(difffile, 'w') - else: - outfh = sys.stdout - - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if 'opposite' in outputoptions.lower(): - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if 'notcommon' in outputoptions.lower(): - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - - if difffile: - outfh.close() - return 0 - except Exception as e: - print("Error while determining differences:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) From 6d884873efdf811f822ca97b93a8b1dda4985012 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 6 Jan 2020 15:54:48 +0000 Subject: [PATCH 51/54] GIS-1621 - Stop showing long git status messages --- versioncontrol/git.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 751d1c8..f5b2ebd 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -80,7 +80,8 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status" + echo "Found ${gitURL} checked out at ${reporootsubfolder}" +/* dir(reporootsubfolder) { if (isUnix()) { sh """ @@ -94,6 +95,7 @@ def String checkouttoreporoot(String gitURL) { """ } } +*/ } } return reporootsubfolder From 264d434404ca68ca93e1d64e6ff0b2ca1defcebc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 7 Jan 2020 14:32:24 +0000 Subject: [PATCH 52/54] GIS-1621 - SMB -> EMB --- copy/logic.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 4d5291c..d1ece6d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,6 +36,8 @@ pipeline { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ rm -v -rf meta + # The following is removing a deprecated folder + rm -v -rf diff mv -v "logic"/* . """ archiveArtifacts "${artifactlist}" From 78973161434389f8a1b424f35b4e31910b0c1400 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 9 Jan 2020 15:34:13 +0000 Subject: [PATCH 53/54] CORE-76 - Change version numbers from 1.xxxxx to 0.xxxxx --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 6 +++--- versionstamping/versionstamper.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 96c9626..f600155 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,7 +157,7 @@ class BuildStampInfo extends GISJsonSerialised { int revision String buildstamp - BuildStampInfo(script, buildstampprefix = "1.30000.0.", revision = 0, buildstamp = "1.30000.0.0") { + BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index de60b66..71d7bf6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -67,14 +67,14 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() - def defaultversionprefix = "1.30000.0." + def defaultversionprefix = "0.30000.0." def splits = sourcebranch.split('/') if (splits.size() < 1) { echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" return defaultversionprefix } - def mastertypeversionprefix = "1.20000.0." + def mastertypeversionprefix = "0.20000.0." sourcebranch = splits.last() if (sourcebranch.toLowerCase().find("master") != null) { echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" @@ -83,7 +83,7 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { def splits2 = sourcebranch.split('\\.') if (splits2.size() < 1) { - def thirdpartyversionprefix = "1.25000.0." + def thirdpartyversionprefix = "0.25000.0." echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" return thirdpartyversionprefix } diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 1a65e6d..441ad06 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -8,7 +8,7 @@ def main(argv): # Call once arguments to allow or disallow something parser.add_argument('-f', '--filetoconvert', help='The file to convert') - parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace') parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') args = parser.parse_args(argv) From afd724a91f779203a9267f311a2d1a136b122b67 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Jan 2020 17:04:41 +0000 Subject: [PATCH 54/54] Output repositories associated with sub-components --- release/do.groovy | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/release/do.groovy b/release/do.groovy index a9f0a3e..817fd5f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -4,6 +4,8 @@ unstablereason = "" summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" +repoandtagsfile = "${summaryfolder}/repoandtags.txt" +repoandtagsstring = "" releaserepofolder = "${WORKSPACE}/releaserepos" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" @@ -28,6 +30,8 @@ def preconditions() { unstablereason = "" summarystring = "" writeFile(file: summaryfile, text: "Release summary: \n") + repoandtagsstring = "" + writeFile(file: repoandtagsfile, text: "Repository summary: \n") } def setunstable(String reason) { @@ -42,6 +46,12 @@ def summaryappend(String summary) { summarystring += summary } +def repoandtagsappend(String summary) { + def String repoandtagscontents = readFile(file: repoandtagsfile) + writeFile(file: repoandtagsfile, text: repoandtagscontents + summary) + repoandtagsstring += summary +} + def getpreviousreleasemeta(buildmeta, String checkoutfolder) { def previousreleasemeta = buildmeta.getNewMetaObject(this) // See if there is meta data file from the current release @@ -55,6 +65,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) { def checkoutreleaserepo(buildmeta, String checkoutfolder) { def String releaserepo = buildmeta.releaseinfo.releaserepoURL // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + repoandtagsappend(" Release repo: ${releaserepo}") def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { sh("git clone ${releaserepo} ${checkoutfolder}") @@ -70,6 +81,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl if (sourcerepo) { + repoandtagsappend(" Source repo: ${sourcerepo}") // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component if (fileExists(checkoutfolder)) { dir(checkoutfolder) { @@ -89,6 +101,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { sh("git clone ${sourcerepo} ${checkoutfolder}") } } + } else { + repoandtagsappend(" Source repo: None") } } @@ -365,6 +379,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) @@ -374,6 +389,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri } summaryappend("\n") + repoandtagsappend("\n") } def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) {