From 95caf56875baa6ed8b58fd36c7e9cf9855a3ea81 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 13:28:28 +0000 Subject: [PATCH 1/8] Make installer build use meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 643e203..15f77d0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -605,8 +605,8 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), - ArrayList releasedfromsubmeta = new ArrayList(), + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { def String artifactlist = METAARTIFACTS From a5cd7e43acf791622620f270e12decfe2953ff3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 14:47:13 +0000 Subject: [PATCH 2/8] Import specific job binaries; Add imported meta to this jobs meta --- import/importer.groovy | 6 ++++-- install/installhelper.groovy | 19 +++++-------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..bf040ce 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,7 +1,8 @@ -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, + excludes: excludes, fingerprintArtifacts: fingerprint, projectName: jenkinsjob.trim(), selector: specific(buildnumber.toString()), @@ -10,9 +11,10 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St return meta.importrelated(destpath) } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, + excludes: excludes, fingerprintArtifacts: fingerprint, projectName: jenkinsjob.trim(), selector: [ diff --git a/install/installhelper.groovy b/install/installhelper.groovy index f523799..fdd6016 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,30 +1,21 @@ // Given details about artifacts to import and details of installers to build, import artifacts that are required -def artifactsImporter(artifactDetails, installerDetails) { +def artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] - //echo "To build: ${installer} use artifactKey: ${artifactKey}" - (sourceProject, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { - //echo "Artifact info: ${artifactInfo}" - copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" - def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" - def folderToCopy = "${destFolder}\\version" - echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" - fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) - //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) - def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - artifactVersion = gh.getSHAFromVersion("${destFolder}/version") - artifactFetched.put(artifactKey, artifactVersion) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destpath, toCopy, notToCopy, true) } } return artifactFetched } // Export any required binaries to the /bin folder -def exportBinaries(artifactDetails, installerDetails) { +def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { From 5aae28adceb6c3d13281be015d9f9519b3e34262 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 15:06:26 +0000 Subject: [PATCH 3/8] Import specific job binaries; Add imported meta to this jobs meta --- install/installhelper.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index fdd6016..206a6c1 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -6,9 +6,8 @@ def artifactsImporter(meta, artifactDetails, installerDetails) { for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) - env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destpath, toCopy, notToCopy, true) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) } } return artifactFetched From 2d679105c8c4f23cf7e46940d8547c36d9e99351 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 15:36:06 +0000 Subject: [PATCH 4/8] Import specific job binaries; Add imported meta to this jobs meta --- install/installhelper.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 206a6c1..0155c7f 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,5 +1,5 @@ // Given details about artifacts to import and details of installers to build, import artifacts that are required -def artifactsImporter(meta, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") @@ -8,6 +8,7 @@ def artifactsImporter(meta, artifactDetails, installerDetails) { (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) + artifactFetched.put(artifactKey, importmeta) } } return artifactFetched From fa3cc37a4d9ec6c79c55aaa09e2772aea9065977 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 16:47:07 +0000 Subject: [PATCH 5/8] Post subtree pull corrections --- publish/do.groovy | 7 +++- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/do.groovy | 14 ++++++- release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From 8ae90886986d45e2261d7a1674737241b85298a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:38:54 +0000 Subject: [PATCH 6/8] Fixing version stamping in windows --- versionnumbering/do.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 9b4a42e..0c53ce9 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -1,17 +1,20 @@ def stamp(convertdetails) { for (convertdetail in convertdetails) { - filetoconvert = convertdetail[0] - pattern = convertdetail[1] - replacement = convertdetail[2] + def filetoconvert = convertdetail[0] + def pattern = convertdetail[1] + def replacement = convertdetail[2] + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" + echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ - "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } else { bat """ - "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + "%${PYTHON}%" -u "%${stampscript}%" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" """ } } From fc461ba62b9caf0e2f7d1c8215f7aa309ee45a4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:46:44 +0000 Subject: [PATCH 7/8] Fixing version stamping in windows --- versionnumbering/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 0c53ce9..9f45f70 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -14,7 +14,7 @@ def stamp(convertdetails) { } else { bat """ - "%${PYTHON}%" -u "%${stampscript}%" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } } From 7bd8655a9224716056a3ec25beed4d4ba97ad6b3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:59:26 +0000 Subject: [PATCH 8/8] Fixing version stamping in windows --- versionnumbering/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 9f45f70..666aee3 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -4,9 +4,9 @@ def stamp(convertdetails) { def filetoconvert = convertdetail[0] def pattern = convertdetail[1] def replacement = convertdetail[2] - echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + //echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - echo "Using python: ${PYTHON} and script: ${stampscript}" + //echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}"