From 253025093e58bdfab2330ba1481f9659d8ac6d95 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 1 Apr 2020 13:09:34 +0100 Subject: [PATCH 01/15] Start converting autobuild for fx3 --- textutils/subst.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textutils/subst.groovy b/textutils/subst.groovy index 055a269..c0207a3 100644 --- a/textutils/subst.groovy +++ b/textutils/subst.groovy @@ -1,5 +1,5 @@ -def substitute(String filepath, String pattern, String subst) { +def substitute(String filepath, pattern, String subst) { fileText = readFile(file: filepath, encoding: 'UTF-8') def converted = fileText.replaceAll(pattern, subst) writeFile(file: filepath, text: converted, encoding: 'UTF-8') From ef343d40872aaca9649cccd9809b8002dc9c2c08 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 6 Apr 2020 10:04:59 +0100 Subject: [PATCH 02/15] GIS-7336 - Bring in FX3 into PS pipeline --- copy/all.groovy | 70 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 copy/all.groovy diff --git a/copy/all.groovy b/copy/all.groovy new file mode 100644 index 0000000..c492b60 --- /dev/null +++ b/copy/all.groovy @@ -0,0 +1,70 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + IMPORTS = 'imports' + cleanfolders = [ + IMPORTS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import source artifacts') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, SOURCE_ARTIFACT_LOCATION, SOURCE_ARTIFACT_BUILD_JOB, SOURCE_ARTIFACT_BUILD_SELECTOR, SOURCE_ARTIFACT_RELEASE_REPO, SOURCE_ARTIFACT_RELEASE_REVISION, IMPORTS) + } + } + } + stage('Copy source artifacts') { + steps { + script { + dir(IMPORTS) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + archiveArtifacts( + artifacts: '**/*', + excludes: "${IMPORTS}/*,autobuild/*" + ) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} \ No newline at end of file From b68652a3085caa30b23b27d75d597601806c2034 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 6 Apr 2020 10:24:01 +0100 Subject: [PATCH 03/15] GIS-7336 - Bring in FX3 into PS pipeline --- copy/all.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/copy/all.groovy b/copy/all.groovy index c492b60..260a0a1 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -45,10 +45,7 @@ pipeline { stage('Archiving artifacts') { steps { script { - archiveArtifacts( - artifacts: '**/*', - excludes: "${IMPORTS}/*,autobuild/*" - ) + archiveArtifacts artifacts: '**/*', excludes: "${IMPORTS}/*" } } } From de89ba93597c7d2bf321a26f072b53f4fdd94d72 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 6 Apr 2020 10:29:05 +0100 Subject: [PATCH 04/15] GIS-7336 - Bring in FX3 into PS pipeline --- copy/all.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/copy/all.groovy b/copy/all.groovy index 260a0a1..a185570 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -45,7 +45,10 @@ pipeline { stage('Archiving artifacts') { steps { script { - archiveArtifacts artifacts: '**/*', excludes: "${IMPORTS}/*" + def includes = '**/*' + def excludes = "${IMPORTS}/*" + echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" + archiveArtifacts artifacts: includes, excludes: excludes } } } From d7182ae51c86f9554de7d68e16c16fb361bb52d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 6 Apr 2020 10:35:33 +0100 Subject: [PATCH 05/15] GIS-7336 - Bring in FX3 into PS pipeline --- copy/all.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/copy/all.groovy b/copy/all.groovy index a185570..30317a7 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -48,7 +48,10 @@ pipeline { def includes = '**/*' def excludes = "${IMPORTS}/*" echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" - archiveArtifacts artifacts: includes, excludes: excludes + // This does not work: archiveArtifacts artifacts: includes, excludes: excludes + // so + cleaner.clean(cleanfolders) + archiveArtifacts artifacts: includes } } } From 2d53c1761ae0a505589e4e151612515aa3499df8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 6 Apr 2020 10:37:36 +0100 Subject: [PATCH 06/15] GIS-7336 - Bring in FX3 into PS pipeline --- copy/all.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/copy/all.groovy b/copy/all.groovy index 30317a7..a3c6c32 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -50,6 +50,10 @@ pipeline { echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" // This does not work: archiveArtifacts artifacts: includes, excludes: excludes // so + cleanfolders = [ + IMPORTS, + 'autobuild', + ] cleaner.clean(cleanfolders) archiveArtifacts artifacts: includes } From 7120ea56ce93dd429ed548391a32bfe56dfaed99 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 13:45:52 +0100 Subject: [PATCH 07/15] Add flatten option to job artifact option --- import/importer.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/import/importer.groovy b/import/importer.groovy index 41f5118..a4f3224 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -25,6 +25,7 @@ class ImportOptions { String excludes = '', boolean fingerprint = false, boolean nested = false, + boolean flatten = false, appendtoparentmeta = true) { m_script = script m_sourceselect = sourceselect.trim() @@ -33,6 +34,7 @@ class ImportOptions { m_excludes = excludes.trim() m_fingerprint = fingerprint m_nested = nested + m_flatten = flatten m_appendtoparentmeta = appendtoparentmeta } @@ -71,6 +73,10 @@ class ImportOptions { m_nested = nested } + void flattenimports(boolean flatten) { + m_flatten = flatten + } + void setappendtoparentmeta(boolean appendtoparentmeta) { m_appendtoparentmeta = appendtoparentmeta } @@ -113,6 +119,7 @@ def artifactimportfrombuildjob(meta, importoptions, String destpath) { $class: 'ParameterizedBuildSelector', parameterName: importoptions.m_buildnumberorselector ], + flatten: importoptions.m_flatten, target: destpath ) def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) From bfa48433b643418c9b65bc396278a6a3b67d2182 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 14:31:12 +0100 Subject: [PATCH 08/15] Correct workspace clean when autobuild scripts in a custom position --- autobuilder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ccc32c1..d78eab0 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,14 +1,14 @@ // Assume that WORKSPACE environment variable is set -autobuildroot = "${WORKSPACE}/autobuild" +autobuildroot = 'autobuild' commonscriptsroot = "${autobuildroot}/Common_scripts" metafolderscript = null def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { - if (doclean) { + if (doclean) { echo "Cleaning workspace" // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - stash allowEmpty: true, includes: 'autobuild/**', name: 'Clean workspace stash', useDefaultExcludes: false + stash allowEmpty: true, includes: "${autobuildroot}/**', name: 'Clean workspace stash', useDefaultExcludes: false cleanWs() unstash name: 'Clean workspace stash' } From 0e9186fc4a362fb024521a622aa970b3c9a08692 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 14:35:05 +0100 Subject: [PATCH 09/15] Correct workspace clean when autobuild scripts in a custom position --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index d78eab0..4ad6acc 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -8,7 +8,7 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - stash allowEmpty: true, includes: "${autobuildroot}/**', name: 'Clean workspace stash', useDefaultExcludes: false + stash allowEmpty: true, includes: "${autobuildroot}/**", name: 'Clean workspace stash', useDefaultExcludes: false cleanWs() unstash name: 'Clean workspace stash' } From 9698d245fceeea9c49f12400d9dacff950441c71 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 15:14:25 +0100 Subject: [PATCH 10/15] Cleanws stash is not saving .git folder --- autobuilder.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 4ad6acc..71d9b6a 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -8,9 +8,14 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - stash allowEmpty: true, includes: "${autobuildroot}/**", name: 'Clean workspace stash', useDefaultExcludes: false + def stashname = 'Clean workspace stash' + stash(allowEmpty: true, + includes: "${autobuildroot}/**,${autobuildroot}/.git/**", + name: stashname, + useDefaultExcludes: false + ) cleanWs() - unstash name: 'Clean workspace stash' + unstash name: stashname } else { echo "Not cleaning workspace" From b131bc7218547b71f8598b2db56deff0f39deeba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 15:21:16 +0100 Subject: [PATCH 11/15] Correct import flattenning --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index a4f3224..650a7a8 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -9,6 +9,7 @@ class ImportOptions { String m_excludes // Exclude files/paths/folders that match the exclude filter boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_flatten // Flatten files and folders imported boolean m_appendtoparentmeta // Append to parent meta String m_sourceselect // Where are the artifacts coming from From e8258c9b283c9af2b785fff5a74c4e96e36452a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 17:59:47 +0100 Subject: [PATCH 12/15] Correct import flattenning --- import/importer.groovy | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 650a7a8..bad2f5a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -10,6 +10,7 @@ class ImportOptions { boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts boolean m_nested // Import all child artifacts too (identified in the related meta info boolean m_flatten // Flatten files and folders imported + boolean m_includemetainfilter // Make sure meta is also included in filter boolean m_appendtoparentmeta // Append to parent meta String m_sourceselect // Where are the artifacts coming from @@ -27,6 +28,7 @@ class ImportOptions { boolean fingerprint = false, boolean nested = false, boolean flatten = false, + boolean includemetainfilter = true, appendtoparentmeta = true) { m_script = script m_sourceselect = sourceselect.trim() @@ -36,6 +38,7 @@ class ImportOptions { m_fingerprint = fingerprint m_nested = nested m_flatten = flatten + m_includemetainfilter = includemetainfilter m_appendtoparentmeta = appendtoparentmeta } @@ -77,6 +80,10 @@ class ImportOptions { void flattenimports(boolean flatten) { m_flatten = flatten } + + void includemetainfilter(boolean includemetainfilter) { + m_includemetainfilter = includemetainfilter + } void setappendtoparentmeta(boolean appendtoparentmeta) { m_appendtoparentmeta = appendtoparentmeta @@ -105,6 +112,14 @@ ImportOptions getNewImportOptions() { return new ImportOptions(this) } +def formFilter(ImportOptions importoptions) { + String filter = importoptions.m_filter + if (importoptions.m_includemetainfilter) { + filter = 'info/**,' + filter + } + return filter +} + def artifactimportfrombuildjob(meta, importoptions, String destpath) { if (importoptions.m_sourceselect != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") @@ -112,7 +127,7 @@ def artifactimportfrombuildjob(meta, importoptions, String destpath) { } copyArtifacts( - filter: importoptions.m_filter, + filter: formFilter(importoptions), excludes: importoptions.m_excludes, fingerprintArtifacts: importoptions.m_fingerprint, projectName: importoptions.m_jobname, @@ -146,7 +161,7 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { def rootrevision = importoptions.m_revision echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) @@ -190,7 +205,7 @@ def artifactimportfromrepo(meta, importoptions, String destpath) { def rootrevision = importoptions.m_revision echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) From e0f0ea2e1db7d4cec4c34b023f1ce1357d117395 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 18:06:01 +0100 Subject: [PATCH 13/15] Correct import flattenning --- import/importer.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index bad2f5a..1836b1d 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -115,8 +115,9 @@ ImportOptions getNewImportOptions() { def formFilter(ImportOptions importoptions) { String filter = importoptions.m_filter if (importoptions.m_includemetainfilter) { - filter = 'info/**,' + filter + filter = 'meta/**,' + filter } + echo "Import options filter is: ${filter}" return filter } From bb3a79b92232270eec86f9f18afd532ad8974c8c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 21:09:39 +0100 Subject: [PATCH 14/15] Correct import filter --- import/importer.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 1836b1d..adef629 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -113,9 +113,11 @@ ImportOptions getNewImportOptions() { } def formFilter(ImportOptions importoptions) { - String filter = importoptions.m_filter - if (importoptions.m_includemetainfilter) { - filter = 'meta/**,' + filter + def String filter = importoptions.m_filter + if (filter != '') { + if (importoptions.m_includemetainfilter) { + filter = 'meta/**,' + filter + } } echo "Import options filter is: ${filter}" return filter From a5cfe14f9b5db5cce4d0c2c5404b76beb3543fa5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 Apr 2020 21:25:05 +0100 Subject: [PATCH 15/15] Clean folders before importing by default --- import/importer.groovy | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index adef629..8624421 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -11,6 +11,7 @@ class ImportOptions { boolean m_nested // Import all child artifacts too (identified in the related meta info boolean m_flatten // Flatten files and folders imported boolean m_includemetainfilter // Make sure meta is also included in filter + boolean m_clean // Clean import folder before obtaining artifacts boolean m_appendtoparentmeta // Append to parent meta String m_sourceselect // Where are the artifacts coming from @@ -29,6 +30,7 @@ class ImportOptions { boolean nested = false, boolean flatten = false, boolean includemetainfilter = true, + boolean clean = true, appendtoparentmeta = true) { m_script = script m_sourceselect = sourceselect.trim() @@ -39,6 +41,7 @@ class ImportOptions { m_nested = nested m_flatten = flatten m_includemetainfilter = includemetainfilter + m_clean = clean m_appendtoparentmeta = appendtoparentmeta } @@ -57,7 +60,7 @@ class ImportOptions { } } - void setincrementversionnumber(boolean incrementversionnumber) { + void setincrementversionnumber(boolean incrementversionnumber = true) { m_incrementversionnumber = incrementversionnumber } @@ -69,22 +72,26 @@ class ImportOptions { m_excludes = excludes.trim() } - void setfingerprint(boolean fingerprint) { + void setfingerprint(boolean fingerprint = true) { m_fingerprint = fingerprint } - void setnested(boolean nested) { + void setnested(boolean nested = true) { m_nested = nested } - void flattenimports(boolean flatten) { + void flattenimports(boolean flatten = true) { m_flatten = flatten } - void includemetainfilter(boolean includemetainfilter) { + void includemetainfilter(boolean includemetainfilter = true) { m_includemetainfilter = includemetainfilter } + void clean(boolean clean = true) { + m_clean = clean + } + void setappendtoparentmeta(boolean appendtoparentmeta) { m_appendtoparentmeta = appendtoparentmeta } @@ -123,12 +130,23 @@ def formFilter(ImportOptions importoptions) { return filter } +def conditionalClean(ImportOptions importoptions, String destpath) { + if (importoptions.m_clean) { + echo "Cleaning ${destpath}" + dir(destpath) { + deleteDir() + } + } +} + def artifactimportfrombuildjob(meta, importoptions, String destpath) { if (importoptions.m_sourceselect != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") return } + conditionalClean(importoptions, destpath) + copyArtifacts( filter: formFilter(importoptions), excludes: importoptions.m_excludes, @@ -164,6 +182,8 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { def rootrevision = importoptions.m_revision echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + conditionalClean(importoptions, destpath) + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) @@ -204,6 +224,8 @@ def artifactimportfromrepo(meta, importoptions, String destpath) { return } + conditionalClean(importoptions, destpath) + def rooturl = importoptions.m_repo def rootrevision = importoptions.m_revision echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}"