From 49a38e0eb1930649c7aa61a4cda9f64cb23a0f42 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 12 Apr 2019 14:55:49 +0100 Subject: [PATCH 01/11] Remove diff folder before building --- copy/logic.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..0b4823d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -14,6 +14,7 @@ pipeline { cleanfolders = [ "${WORKSPACE}/logic", "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/diff", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) From 3286473a46b80cb90aaeba9fd13c24e13a540821 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 15 Apr 2019 09:57:18 +0100 Subject: [PATCH 02/11] Logic copy doesn't work if pre-existing folders from previous builds exist --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..5a0c80d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,7 +36,7 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf meta + rm -v -rf diff meta embedded_logic mv -v logic/* . """ archiveArtifacts "${artifactlist}" From 46105a841d748708cb18a9627d7081744ce5964d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Apr 2019 22:47:29 +0100 Subject: [PATCH 03/11] GIS-7565 - Attempt at altering artifact output structure in order to simplify publishable item presentation to CAMs at publishing time --- release/do.groovy | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index fd2b06b..596f107 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -275,6 +275,22 @@ def sourcegitpush(String dryrunpush ) { } } +def boolean isalreadyreleased(buildmeta, tag) { + def boolean alreadyreleased = buildmeta.releaseinfo.released + if (alreadyreleased) { + echo "Component is already marked as released: ${alreadyreleased}" + } else { + String commitshacode = releasegitgetshacode(tag) + if (commitshacode) { + echo "Tag: ${tag} already exists in this repo therefore marking as already released" + alreadyreleased = true + } else { + alreadyreleased = false + } + } + return alreadyreleased +} + def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") def String buildstamptag = buildmeta.releasetag() @@ -283,8 +299,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - def boolean alreadyreleased = buildmeta.releaseinfo.released - echo "Already released: ${alreadyreleased}" + def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) def String postcommitshacode def boolean changed = false From 8cadf19ce95e36b8c6b4a89409d62d722a64ca06 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Apr 2019 18:30:16 +0100 Subject: [PATCH 04/11] Slight change of message --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 596f107..9b46114 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -189,7 +189,7 @@ def boolean releasegittag(String postcommitshacode, String tag) { if (existingshacode) { // Tag exists if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + setunstable("The release tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") summaryappend("(${tag} - Could not add!)") } else { summaryappend("(${tag} - Already added)") From cbacd69e00c09bc24c21f473bbbbff9de27f1cc3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 09:38:53 +0100 Subject: [PATCH 05/11] Perform a git lfs install each time there is a git checkout --- versioncontrol/git.groovy | 58 +++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 253ffee..b386a9a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -75,7 +75,23 @@ def String checkouttoreporoot(String gitURL) { return reporootsubfolder } +def checkgitlfsinstall() { + if (isUnix()) { + sh """ + git lfs install + if [ \$? -ne 0 ]; then exit 1; fi + """ + + } else { + bat """ + git lfs install + if errorlevel 1 EXIT /b 1 + """ + } +} + def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', @@ -99,33 +115,33 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } String getSHAFromVersion(String versionFolder) { - dir (versionFolder) { + dir (versionFolder) { srcRevision = readFile("gitrevision.txt").trim() - } - return srcRevision + } + return srcRevision } boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def keys = versions.keySet() - if (keys.size() < 1) { - return false - } - if (keys.size() == 1) { - return true - } + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } else { - version = "" - for (key in keys) { - nextversion = versions.get(key) - if (version == "") { - version = nextversion - } - if (nextversion != version) { - return false - } - } - } - return true + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true } def stripgithistory(String rootfolder) { From e12939d9287e9a8f24c6ebb83d3a93a4007fb523 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 11:56:15 +0100 Subject: [PATCH 06/11] Added a comment --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 9b46114..b3d54c7 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -61,7 +61,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { - // Clear all existing files + // Clear all existing files - Note that this does not remove .git* sh("git rm -r *") } return previousreleasemeta From e00f228c7b9a5bdefa3eeefa1e929f473e4d2591 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 14:20:28 +0100 Subject: [PATCH 07/11] Detect if a publishable item filepath is valid or not --- publishing/utils.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 145a7e9..2ab4059 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,8 +1,23 @@ + +// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout +def checkallowed(String filepath) { + if (filepath.endsWith('/**') || filepath.endsWith('/')) { + def files = findFiles glob: filepath + if (files.length > 0) { + return + } else { + error "There are no files/folders under the publishable item: ${filepath}" + } + } + error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" +} + // Generate a list of artifact files that are publishable to outside parties def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { + checkallowed(filepath) meta.addPublishable(filepath) } } From 8fecc6c3200bb0643b33ac9b44b78f43b7545cc6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 14:50:31 +0100 Subject: [PATCH 08/11] Move functionality to common area --- install/installhelper.groovy | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 1464e9d..32da99b 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,3 +1,27 @@ +import hudson.FilePath + +// Transfer file from intermediate position to final position; +// creating necessary folders on the way; +// deleting intermediate file +def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { + def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() + + def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" + def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" + def export_filepath = new hudson.FilePath(channel, abs_export_out) + def export_out_folder = export_filepath.getParent() + def export_out_folder_string = export_out_folder.toString() + def export_out_file = export_filepath.getName() + + echo "Creating folder ${export_out_folder_string}" + export_out_folder.mkdirs() + + echo "Copying from ${pathToIntermediate} to ${abs_export_out}" + def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) + intermediate_filepath.copyTo(export_filepath) + intermediate_filepath.delete() +} + // Given details about artifacts to import and details of installers to build, import artifacts that are required def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] From c4651ca04e4c18cacb292be662a19681712b11b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 15:13:20 +0100 Subject: [PATCH 09/11] Release list is not being passed correctly --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6e0823e..b16db5d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -233,6 +233,7 @@ class ReleaseInfo extends GISJsonSerialised { } void addReleaseItem(String releaseitem) { + script.echo("Adding release item: ${releaseitem} to release items") releaseitems.add(releaseitem) } From 91a58544c3e8af3870b1200270c8cf90c5174b94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 15:36:29 +0100 Subject: [PATCH 10/11] Remove redundant binaries copy to bin folder stage --- install/installhelper.groovy | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 32da99b..eebbd34 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -38,22 +38,6 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { return artifactFetched } -// Export any required binaries to the /bin folder -def placeBinaries(artifactDetails, installerDetails) { - for (installer in toBuild) { - (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) - if (exportBinaries) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) - for (binaryFolder in binariesFolders) { - def binariesSourceFolder = "${destFolder}/${binaryFolder}" - def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" - echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" - fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) - } - } - } -} - @NonCPS def checkISMfilebackend(String xml) { def String message = "" From 730ec0c5ff05fcec74bc1b7b1c432d07f9058df5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Apr 2019 10:55:10 +0100 Subject: [PATCH 11/11] Don't check publishable file paths for meta/** as its only save usually at the end of the build --- publishing/utils.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 2ab4059..2daf6f6 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,7 +1,7 @@ // The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout -def checkallowed(String filepath) { +def allowed(String filepath) { if (filepath.endsWith('/**') || filepath.endsWith('/')) { def files = findFiles glob: filepath if (files.length > 0) { @@ -17,7 +17,9 @@ def checkallowed(String filepath) { def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { - checkallowed(filepath) + if (filepath != meta.METAARTIFACTS) { + allowed(filepath) + } meta.addPublishable(filepath) } }