From 67785ac859fb5e376be83a6e63e1d4b22007f410 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Apr 2019 17:14:15 +0100 Subject: [PATCH 001/177] Use the jenkins pipeline checkout mechanism for getting release and source repo --- release/do.groovy | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index fd2b06b..ada51bb 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -10,6 +10,7 @@ tempfolder = "${WORKSPACE}/temp" gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" metascript = autobuilder.getmetafolderscript() +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -57,7 +58,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - sh("git clone ${releaserepo} ${checkoutfolder}") + gh.gitCheckout(releaserepo, '*/master', checkoutfolder, "", 1, false) } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { @@ -69,27 +70,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl - if (sourcerepo) { - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } - } + gh.gitCheckout(sourcerepo, '*/master', checkoutfolder, "", 1, false) } def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { From fb7953a95909ad06a75d2f4e890a866811963482 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Apr 2019 17:24:13 +0100 Subject: [PATCH 002/177] Revert "Use the jenkins pipeline checkout mechanism for getting release and source repo" This reverts commit 6808492ff5bc48c3e5d6a3e4239c0c46b9d0145e. --- release/do.groovy | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index ada51bb..fd2b06b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -10,7 +10,6 @@ tempfolder = "${WORKSPACE}/temp" gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" metascript = autobuilder.getmetafolderscript() -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -58,7 +57,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - gh.gitCheckout(releaserepo, '*/master', checkoutfolder, "", 1, false) + sh("git clone ${releaserepo} ${checkoutfolder}") } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { @@ -70,7 +69,27 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl - gh.gitCheckout(sourcerepo, '*/master', checkoutfolder, "", 1, false) + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } } def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { From c9e2557aa9c7b48a11602393b53842ba5e23cfe3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 15:53:44 +0100 Subject: [PATCH 003/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 45 +++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 819a580..d064161 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -30,4 +30,47 @@ def placeBinaries(artifactDetails, installerDetails) { } } -return this; \ No newline at end of file +def checkISMfile(String ISMfile) { + def xmlFile = new File(ISMfile) + + def parsedxml = new XmlParser().parse(xmlFile) + def pathvariableinfo = [:] + def exitcode = 0 + // Load up a map of path variable and path pairs + // Ignore blank path definitions + // Ignore the workspace definition which should have the only absolute path + parsedxml.each{ table -> + if (table.@name == "ISPathVariable") { + table.row.each{ row -> + def pathname = row.td[0].value()[0] + def path = row.td[1].value()[0] + if (path && pathname != "WORKSPACE") { + pathvariableinfo[pathname] = path + } + } + } + } + // Check that each path variable is defined beginning with a path variable or the WORKSPACE one + pathvariableinfo.each{ pair -> + println "pair key: ${pair.key} value: ${pair.value}" + def path = pair.value + def pathvarmatcher = path =~ /<(.*)>.*/ + if (pathvarmatcher.matches()) { + def pathvarfound = pathvarmatcher.group(1) + if (pathvarfound != 'WORKSPACE') { + if (!pathvariableinfo.containsKey(pathvarfound)) { + println "Undefined path variable ${pathvarfound} found" + exitcode = 1 + } + } + } else { + println "Path: ${path} does not begin with a path variable" + exitcode = 1 + } + } + if (exitcode > 0) { + error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" + } +} + +return this; From eee2111033b7dd9ae16c553277a7f069f7782e3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 17:58:05 +0100 Subject: [PATCH 004/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index d064161..886c604 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -31,9 +31,9 @@ def placeBinaries(artifactDetails, installerDetails) { } def checkISMfile(String ISMfile) { - def xmlFile = new File(ISMfile) + def xml = readFile(ISMfile) - def parsedxml = new XmlParser().parse(xmlFile) + def parsedxml = new XmlParser().parseText(xml) def pathvariableinfo = [:] def exitcode = 0 // Load up a map of path variable and path pairs From fb6c1a0e31c5f112a449ad5f4c46bdc1e8d0dd46 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:07:31 +0100 Subject: [PATCH 005/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 886c604..6cecb85 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -33,7 +33,7 @@ def placeBinaries(artifactDetails, installerDetails) { def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) - def parsedxml = new XmlParser().parseText(xml) + def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] def exitcode = 0 // Load up a map of path variable and path pairs From 1fce006fe33c5932874df279d98bc391fb85ea2b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:21:53 +0100 Subject: [PATCH 006/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6cecb85..f3dcdd4 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -30,6 +30,7 @@ def placeBinaries(artifactDetails, installerDetails) { } } +@NonCPS def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) From 865deaecf843177e6f223ef3c676dcbbc1d96601 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:30:16 +0100 Subject: [PATCH 007/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index f3dcdd4..823c117 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -33,7 +33,8 @@ def placeBinaries(artifactDetails, installerDetails) { @NonCPS def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) - + echo "Checking ISM file: ${ISMfile}" + def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] def exitcode = 0 @@ -53,24 +54,26 @@ def checkISMfile(String ISMfile) { } // Check that each path variable is defined beginning with a path variable or the WORKSPACE one pathvariableinfo.each{ pair -> - println "pair key: ${pair.key} value: ${pair.value}" + echo "pair key: ${pair.key} value: ${pair.value}" def path = pair.value def pathvarmatcher = path =~ /<(.*)>.*/ if (pathvarmatcher.matches()) { def pathvarfound = pathvarmatcher.group(1) if (pathvarfound != 'WORKSPACE') { if (!pathvariableinfo.containsKey(pathvarfound)) { - println "Undefined path variable ${pathvarfound} found" + echo "Undefined path variable ${pathvarfound} found" exitcode = 1 } } } else { - println "Path: ${path} does not begin with a path variable" + echo "Path: ${path} does not begin with a path variable" exitcode = 1 } } if (exitcode > 0) { error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" + } else { + echo "ISM file: ${ISMfile} - ok" } } From 16ff70565612f9ef6590e8c4c3c928b6ed66e27e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 08:39:15 +0100 Subject: [PATCH 008/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 823c117..51edd3d 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -31,9 +31,8 @@ def placeBinaries(artifactDetails, installerDetails) { } @NonCPS -def checkISMfile(String ISMfile) { - def xml = readFile(ISMfile) - echo "Checking ISM file: ${ISMfile}" +def checkISMfilebackend(String xml) { + def String message = "" def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] @@ -54,26 +53,31 @@ def checkISMfile(String ISMfile) { } // Check that each path variable is defined beginning with a path variable or the WORKSPACE one pathvariableinfo.each{ pair -> - echo "pair key: ${pair.key} value: ${pair.value}" + message += "pair key: ${pair.key} value: ${pair.value}\n" def path = pair.value def pathvarmatcher = path =~ /<(.*)>.*/ if (pathvarmatcher.matches()) { def pathvarfound = pathvarmatcher.group(1) if (pathvarfound != 'WORKSPACE') { if (!pathvariableinfo.containsKey(pathvarfound)) { - echo "Undefined path variable ${pathvarfound} found" + message += "Undefined path variable ${pathvarfound} found\n" exitcode = 1 } } } else { - echo "Path: ${path} does not begin with a path variable" + message += "Path: ${path} does not begin with a path variable\n" exitcode = 1 } } + return [exitcode, message] +} + +def checkISMfile(String ISMfile) { + echo "Checking ISM file: ${ISMfile}" + def xml = readFile(ISMfile) + (exitcode, message) = checkISMfilebackend(xml) if (exitcode > 0) { - error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" - } else { - echo "ISM file: ${ISMfile} - ok" + error "ISM file errors found:\n${message}" } } From 291b39bf5bd2e05374a2fe136ca5476747df62e5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 08:57:32 +0100 Subject: [PATCH 009/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 51edd3d..6ac2e80 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -42,10 +42,10 @@ def checkISMfilebackend(String xml) { // Ignore the workspace definition which should have the only absolute path parsedxml.each{ table -> if (table.@name == "ISPathVariable") { - table.row.each{ row -> - def pathname = row.td[0].value()[0] - def path = row.td[1].value()[0] - if (path && pathname != "WORKSPACE") { + table['row'].each{ row -> + def pathname = row['td'][0].value()[0] + def path = row['td'][1].value()[0] + if (path && pathname != 'WORKSPACE') { pathvariableinfo[pathname] = path } } From 429d901d014ef6cd5669d9c0c078f680bd1c8649 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 09:30:24 +0100 Subject: [PATCH 010/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6ac2e80..ca9c502 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -78,6 +78,9 @@ def checkISMfile(String ISMfile) { (exitcode, message) = checkISMfilebackend(xml) if (exitcode > 0) { error "ISM file errors found:\n${message}" + } else { + echo "Checked ISM file: ${ISMfile} - ok" + echo message } } From 098b0cb8372189e4c30c6b55fe56b0ac4c7725c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 10:44:43 +0100 Subject: [PATCH 011/177] Perform installer file (ISM) checking --- install/installhelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index ca9c502..1464e9d 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -76,11 +76,11 @@ def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) (exitcode, message) = checkISMfilebackend(xml) + echo message if (exitcode > 0) { - error "ISM file errors found:\n${message}" + error "ISM file errors found" } else { - echo "Checked ISM file: ${ISMfile} - ok" - echo message + echo "Checked ISM file - ok" } } From 49a38e0eb1930649c7aa61a4cda9f64cb23a0f42 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 12 Apr 2019 14:55:49 +0100 Subject: [PATCH 012/177] Remove diff folder before building --- copy/logic.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..0b4823d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -14,6 +14,7 @@ pipeline { cleanfolders = [ "${WORKSPACE}/logic", "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/diff", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) From 3286473a46b80cb90aaeba9fd13c24e13a540821 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 15 Apr 2019 09:57:18 +0100 Subject: [PATCH 013/177] Logic copy doesn't work if pre-existing folders from previous builds exist --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..5a0c80d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,7 +36,7 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf meta + rm -v -rf diff meta embedded_logic mv -v logic/* . """ archiveArtifacts "${artifactlist}" From 5daf7c01d8239569439ea5ed3ab0f7e53460e0c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 21 Apr 2019 11:28:51 +0100 Subject: [PATCH 014/177] Meta-diff folder is not being reset at the beginning of a build --- meta/folder.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 346b2ee..6e0823e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -655,6 +655,10 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } + static String difffolder() { + return DIFFFOLDERNAME + } + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" @@ -763,6 +767,11 @@ MetaInfo metareset(script) { dir(mi.folder()) { deleteDir() } + + dir(mi.difffolder()) { + deleteDir() + } + return mi } From 46105a841d748708cb18a9627d7081744ce5964d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Apr 2019 22:47:29 +0100 Subject: [PATCH 015/177] GIS-7565 - Attempt at altering artifact output structure in order to simplify publishable item presentation to CAMs at publishing time --- release/do.groovy | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index fd2b06b..596f107 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -275,6 +275,22 @@ def sourcegitpush(String dryrunpush ) { } } +def boolean isalreadyreleased(buildmeta, tag) { + def boolean alreadyreleased = buildmeta.releaseinfo.released + if (alreadyreleased) { + echo "Component is already marked as released: ${alreadyreleased}" + } else { + String commitshacode = releasegitgetshacode(tag) + if (commitshacode) { + echo "Tag: ${tag} already exists in this repo therefore marking as already released" + alreadyreleased = true + } else { + alreadyreleased = false + } + } + return alreadyreleased +} + def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") def String buildstamptag = buildmeta.releasetag() @@ -283,8 +299,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - def boolean alreadyreleased = buildmeta.releaseinfo.released - echo "Already released: ${alreadyreleased}" + def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) def String postcommitshacode def boolean changed = false From 8cadf19ce95e36b8c6b4a89409d62d722a64ca06 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Apr 2019 18:30:16 +0100 Subject: [PATCH 016/177] Slight change of message --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 596f107..9b46114 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -189,7 +189,7 @@ def boolean releasegittag(String postcommitshacode, String tag) { if (existingshacode) { // Tag exists if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + setunstable("The release tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") summaryappend("(${tag} - Could not add!)") } else { summaryappend("(${tag} - Already added)") From cbacd69e00c09bc24c21f473bbbbff9de27f1cc3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 09:38:53 +0100 Subject: [PATCH 017/177] Perform a git lfs install each time there is a git checkout --- versioncontrol/git.groovy | 58 +++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 253ffee..b386a9a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -75,7 +75,23 @@ def String checkouttoreporoot(String gitURL) { return reporootsubfolder } +def checkgitlfsinstall() { + if (isUnix()) { + sh """ + git lfs install + if [ \$? -ne 0 ]; then exit 1; fi + """ + + } else { + bat """ + git lfs install + if errorlevel 1 EXIT /b 1 + """ + } +} + def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', @@ -99,33 +115,33 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } String getSHAFromVersion(String versionFolder) { - dir (versionFolder) { + dir (versionFolder) { srcRevision = readFile("gitrevision.txt").trim() - } - return srcRevision + } + return srcRevision } boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def keys = versions.keySet() - if (keys.size() < 1) { - return false - } - if (keys.size() == 1) { - return true - } + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } else { - version = "" - for (key in keys) { - nextversion = versions.get(key) - if (version == "") { - version = nextversion - } - if (nextversion != version) { - return false - } - } - } - return true + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true } def stripgithistory(String rootfolder) { From e12939d9287e9a8f24c6ebb83d3a93a4007fb523 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 11:56:15 +0100 Subject: [PATCH 018/177] Added a comment --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 9b46114..b3d54c7 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -61,7 +61,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { - // Clear all existing files + // Clear all existing files - Note that this does not remove .git* sh("git rm -r *") } return previousreleasemeta From e00f228c7b9a5bdefa3eeefa1e929f473e4d2591 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 14:20:28 +0100 Subject: [PATCH 019/177] Detect if a publishable item filepath is valid or not --- publishing/utils.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 145a7e9..2ab4059 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,8 +1,23 @@ + +// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout +def checkallowed(String filepath) { + if (filepath.endsWith('/**') || filepath.endsWith('/')) { + def files = findFiles glob: filepath + if (files.length > 0) { + return + } else { + error "There are no files/folders under the publishable item: ${filepath}" + } + } + error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" +} + // Generate a list of artifact files that are publishable to outside parties def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { + checkallowed(filepath) meta.addPublishable(filepath) } } From 8fecc6c3200bb0643b33ac9b44b78f43b7545cc6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 14:50:31 +0100 Subject: [PATCH 020/177] Move functionality to common area --- install/installhelper.groovy | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 1464e9d..32da99b 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,3 +1,27 @@ +import hudson.FilePath + +// Transfer file from intermediate position to final position; +// creating necessary folders on the way; +// deleting intermediate file +def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { + def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() + + def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" + def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" + def export_filepath = new hudson.FilePath(channel, abs_export_out) + def export_out_folder = export_filepath.getParent() + def export_out_folder_string = export_out_folder.toString() + def export_out_file = export_filepath.getName() + + echo "Creating folder ${export_out_folder_string}" + export_out_folder.mkdirs() + + echo "Copying from ${pathToIntermediate} to ${abs_export_out}" + def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) + intermediate_filepath.copyTo(export_filepath) + intermediate_filepath.delete() +} + // Given details about artifacts to import and details of installers to build, import artifacts that are required def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] From c4651ca04e4c18cacb292be662a19681712b11b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 15:13:20 +0100 Subject: [PATCH 021/177] Release list is not being passed correctly --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6e0823e..b16db5d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -233,6 +233,7 @@ class ReleaseInfo extends GISJsonSerialised { } void addReleaseItem(String releaseitem) { + script.echo("Adding release item: ${releaseitem} to release items") releaseitems.add(releaseitem) } From 91a58544c3e8af3870b1200270c8cf90c5174b94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 15:36:29 +0100 Subject: [PATCH 022/177] Remove redundant binaries copy to bin folder stage --- install/installhelper.groovy | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 32da99b..eebbd34 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -38,22 +38,6 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { return artifactFetched } -// Export any required binaries to the /bin folder -def placeBinaries(artifactDetails, installerDetails) { - for (installer in toBuild) { - (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) - if (exportBinaries) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) - for (binaryFolder in binariesFolders) { - def binariesSourceFolder = "${destFolder}/${binaryFolder}" - def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" - echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" - fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) - } - } - } -} - @NonCPS def checkISMfilebackend(String xml) { def String message = "" From 730ec0c5ff05fcec74bc1b7b1c432d07f9058df5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Apr 2019 10:55:10 +0100 Subject: [PATCH 023/177] Don't check publishable file paths for meta/** as its only save usually at the end of the build --- publishing/utils.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 2ab4059..2daf6f6 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,7 +1,7 @@ // The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout -def checkallowed(String filepath) { +def allowed(String filepath) { if (filepath.endsWith('/**') || filepath.endsWith('/')) { def files = findFiles glob: filepath if (files.length > 0) { @@ -17,7 +17,9 @@ def checkallowed(String filepath) { def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { - checkallowed(filepath) + if (filepath != meta.METAARTIFACTS) { + allowed(filepath) + } meta.addPublishable(filepath) } } From d55d2a24c4ea87e35a3ebae26f402807f7b10819 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:13:08 +0100 Subject: [PATCH 024/177] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- release/registerrepo.groovy | 2 +- versioncontrol/git.groovy | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index 2dffdc7..bd6a528 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -4,7 +4,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix) + vc.checkRepoExists(releaserepo) } meta.releaseinfo.set(releaserepo) } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index b386a9a..96fa571 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -114,6 +114,28 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo ]) } +def checkRepoExists(String gitURL) { + def boolean result = false + if (isUnix()) { + result = sh( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + } else { + result = bash( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + if errorlevel 1 EXIT /b 1 + """, + returnStatus: true + ) + } + if (!result) { + error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } +} + String getSHAFromVersion(String versionFolder) { dir (versionFolder) { srcRevision = readFile("gitrevision.txt").trim() From c13ba0d6289bd14c0a76198ca7125cfa859b3328 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:15:34 +0100 Subject: [PATCH 025/177] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 96fa571..cea31fc 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -124,7 +124,7 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } else { - result = bash( script: """ + result = bat( script: """ git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY if errorlevel 1 EXIT /b 1 """, From 9af654fb28e93396c128a571be4ab365fbd2bd45 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:20:36 +0100 Subject: [PATCH 026/177] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index cea31fc..36f4c64 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -119,14 +119,12 @@ def checkRepoExists(String gitURL) { if (isUnix()) { result = sh( script: """ git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true ) } else { result = bat( script: """ - git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - if errorlevel 1 EXIT /b 1 + @git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY """, returnStatus: true ) From a42acef027fd7fb3b8c4b47c9fb470584a27b75b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:26:22 +0100 Subject: [PATCH 027/177] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 36f4c64..12d5bc6 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -130,6 +130,7 @@ def checkRepoExists(String gitURL) { ) } if (!result) { + echo "git ls-remote return result: ${result}" error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") } } From 7a62550a07890fc69f415e36dcaf5c4f22627dd8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:29:15 +0100 Subject: [PATCH 028/177] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 12d5bc6..ee7e99a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -115,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } def checkRepoExists(String gitURL) { - def boolean result = false + def int result = 0 if (isUnix()) { result = sh( script: """ git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY @@ -129,7 +129,7 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } - if (!result) { + if (result > 0) { echo "git ls-remote return result: ${result}" error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") } From 0216685c99d3ce9c564a6c88fc61fd1f8bb99c22 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:31:49 +0100 Subject: [PATCH 029/177] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ee7e99a..f72fc5a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -129,9 +129,11 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } + echo "git ls-remote return result: ${result}" if (result > 0) { - echo "git ls-remote return result: ${result}" error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } else { + echo "The release repo: ${gitURL} exists" } } From 55a2e2a03444ed9e942316f3785667ba4f338375 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 7 May 2019 10:28:39 +0100 Subject: [PATCH 030/177] Warn of git stripping not being supported in Windows --- versioncontrol/git.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f72fc5a..b639c91 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -170,13 +170,17 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def stripgithistory(String rootfolder) { echo "Stripping git history out of ${rootfolder}" dir(rootfolder) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rf XX - find . -name ".gitignore" | xargs -iXX rm -rf XX - find . -name ".gitmodules" | xargs -iXX rm -rf XX - find . -iname ".gitattributes" | xargs -iXX rm -rf XX - """ + if (isUnix()) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + """ + } else { + echo "Not supporting git stripping in Windows at the moment" + } } } From b18dba3c00b34ce1202609d5947050950679d98b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 21:51:31 +0100 Subject: [PATCH 031/177] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index b639c91..fc15f81 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -93,6 +93,12 @@ def checkgitlfsinstall() { def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) + if (!sparseitemlist.isEmpty()) { + echo "Items in sparse item list: " + for (item in sparseitemlist) { + echo item + } + } checkout([ $class: 'GitSCM', userRemoteConfigs: [ From 4c4958db9b40be65fa611df7a5feb3e820873483 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 22:11:54 +0100 Subject: [PATCH 032/177] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index fc15f81..56d206c 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -94,10 +94,11 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) if (!sparseitemlist.isEmpty()) { - echo "Items in sparse item list: " + def itemstring = "Items in sparse item list:\n" for (item in sparseitemlist) { - echo item + itemstring += item + "\n" } + echo itemstring } checkout([ $class: 'GitSCM', @@ -114,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - sparsecheckoutconfig(sparseitemlist) + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], ], submoduleCfg: [], ]) From 5be5dfce45b25b7544fc9b0012f51905c66581d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 22:14:22 +0100 Subject: [PATCH 033/177] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 56d206c..5a92884 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -115,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'SparseCheckoutPaths', sparsecheckoutconfig(sparseitemlist)], ], submoduleCfg: [], ]) From ddc231ccacd21a672eaf1be65fa30640ba692178 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 22:54:47 +0100 Subject: [PATCH 034/177] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 5a92884..b37bbc7 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -11,12 +11,12 @@ String getGitInfo(String workingSetRootPath) { } @NonCPS -def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { +def ArrayList sparsecheckoutconfig(ArrayList paths) { ArrayList sparsecheckoutpaths = new ArrayList() for (path in paths) { sparsecheckoutpaths.add(new SparseCheckoutPath(path)) } - return new SparseCheckoutPaths(sparsecheckoutpaths) + return sparsecheckoutpaths } def String uniquereporootname(String gitURL) { @@ -115,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparsecheckoutconfig(sparseitemlist)], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], ], submoduleCfg: [], ]) From 2c9113cc4d8045de29f6a9aeb33f9c5501904aab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 23:17:20 +0100 Subject: [PATCH 035/177] Print out items in sparse item list if the list is not empty --- publish/publish.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/publish/publish.groovy b/publish/publish.groovy index b478a5d..b74b72a 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -32,6 +32,15 @@ pipeline { } } } + stage('Test sparse checkout') { + steps { + script { + def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + def ArrayList publishablepaths = ['installers/KM/'] + gh.gitCheckout('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone + } + } + } stage('Get release artifact meta') { steps { script { From 7b50a27c26a0b492ddd17160babbaaa40756ef2d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 23:23:32 +0100 Subject: [PATCH 036/177] Print out items in sparse item list if the list is not empty --- publish/publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index b74b72a..6f9fd44 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -36,7 +36,7 @@ pipeline { steps { script { def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def ArrayList publishablepaths = ['installers/KM/'] + def ArrayList publishablepaths = ['installers/KM/*'] gh.gitCheckout('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone } } From 6b9f624770a3e093ffdde027ed228d9d0880b9d0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 09:11:38 +0100 Subject: [PATCH 037/177] Need an alternative way to checkout because git plugin is not working when using LFS and sparse checkout simultaneously --- versioncontrol/git.groovy | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index b37bbc7..fed039a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -90,6 +90,91 @@ def checkgitlfsinstall() { } } +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + checkgitlfsinstall() + def reporootpath = checkouttoreporoot(gitURL) + if (!sparseitemlist.isEmpty()) { + def itemstring = "Items in sparse item list:\n" + for (item in sparseitemlist) { + itemstring += item + "\n" + } + echo itemstring + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + ], + submoduleCfg: [], + ]) +} + +//No credentials specified +//Cloning the remote Git repository +//Cloning repository git@hydra:print-server-releases/print_server_whole.git +//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 +//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git +//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// No valid HEAD. Skipping the resetting +// > git clean -fdx # timeout=10 +// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_7" +// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 + + +// > git rev-parse --is-inside-work-tree # timeout=10 +// Fetching changes from the remote Git repository +// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// Resetting working tree +// > git reset --hard # timeout=10 +// > git clean -fdx # timeout=10 +// Using shallow fetch with depth 1 +// Fetching upstream changes from git@hydra:releases/print-server-installer.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 +// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 +// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git read-tree -mu HEAD # timeout=10 +// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_3" +// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 + +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) From a6cdcccb123d651b62c93fcc1792c4aaa955fe7b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:21:35 +0100 Subject: [PATCH 038/177] Try to get sparse checkout to work with lfs --- publish/do.groovy | 2 +- versioncontrol/git.groovy | 96 +++++++++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b63b15c..1599ca8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -83,7 +83,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite } if (publishablepaths.size() > 0) { def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone + gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone } else { summaryappend("None ") diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index fed039a..1652e6b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -90,37 +90,6 @@ def checkgitlfsinstall() { } } -def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - checkgitlfsinstall() - def reporootpath = checkouttoreporoot(gitURL) - if (!sparseitemlist.isEmpty()) { - def itemstring = "Items in sparse item list:\n" - for (item in sparseitemlist) { - itemstring += item + "\n" - } - echo itemstring - } - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: revision] - ], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], - ], - submoduleCfg: [], - ]) -} - //No credentials specified //Cloning the remote Git repository //Cloning repository git@hydra:print-server-releases/print_server_whole.git @@ -174,10 +143,12 @@ def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repo // Commit message: "2.10180.7389.21125_3" // > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 -// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { +def String lfsinstallandreporoot(String gitURL) { checkgitlfsinstall() - def reporootpath = checkouttoreporoot(gitURL) + return checkouttoreporoot(gitURL) +} + +def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { if (!sparseitemlist.isEmpty()) { def itemstring = "Items in sparse item list:\n" for (item in sparseitemlist) { @@ -185,6 +156,9 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } echo itemstring } +} + +def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -206,6 +180,60 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo ]) } +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + +def gitinit(String targetDir, String reporootpath) { + if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + if (isUnix()) { + sh """ + git init "${targetDir}" + """ + } else { + bat """ + git init "${targetDir}" + """ + } + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") + } + } +} + +def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { + def String sparseitemliststring = "" + if (!sparseitemlist.isEmpty()) { + for (item in sparseitemlist) { + sparseitemliststring += "${item}," + } + dir(targetDir) { + if (isUnix()) { + sh """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } else { + bat """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } + } + } +} + +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitinit(targetDir, reporootpath) + setlfsfetchinclude(targetDir, sparseitemlist) + } + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + def checkRepoExists(String gitURL) { def int result = 0 if (isUnix()) { From 3686e219763bdaaf9608366e0bc57bfc5e63c4f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:32:18 +0100 Subject: [PATCH 039/177] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1652e6b..a5890f2 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -189,6 +189,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo def gitinit(String targetDir, String reporootpath) { if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + echo "git init ${targetDir}" if (isUnix()) { sh """ git init "${targetDir}" @@ -198,6 +199,7 @@ def gitinit(String targetDir, String reporootpath) { git init "${targetDir}" """ } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" dir("${targetDir}/.git/objects/info") { writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") } @@ -210,6 +212,7 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { for (item in sparseitemlist) { sparseitemliststring += "${item}," } + echo "git config lfs.fetchinclude ${sparseitemliststring}" dir(targetDir) { if (isUnix()) { sh """ @@ -225,6 +228,7 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { } def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) showsparseitemlist(sparseitemlist) if (!sparseitemlist.isEmpty()) { From f29e1beb7b0774d09dbc87d8d5810b3e8ae65bc5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:34:52 +0100 Subject: [PATCH 040/177] Try to get sparse checkout to work with lfs --- publish/publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index 6f9fd44..e4539f1 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -37,7 +37,7 @@ pipeline { script { def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def ArrayList publishablepaths = ['installers/KM/*'] - gh.gitCheckout('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone + gh.gitCheckoutAlt('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone } } } From 1aee93e12163bc61a6dbacadf895674519b2fae0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:42:49 +0100 Subject: [PATCH 041/177] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a5890f2..1bf9cd1 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -199,10 +199,10 @@ def gitinit(String targetDir, String reporootpath) { git init "${targetDir}" """ } - echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" - dir("${targetDir}/.git/objects/info") { - writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") - } + } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") } } From eb8a1005ec7f2e4f62e65a76f9386d72e3514699 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 May 2019 00:02:48 +0100 Subject: [PATCH 042/177] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1bf9cd1..ccbd458 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -227,6 +227,25 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { } } +def gitsparserestore(String targetDir) { + echo "Clean out full fetch; Force a sparse fetch" + dir(targetDir) { + if (isUnix()) { + sh """ + git rm -rf . + git clean -fxd + git reset --hard + """ + } else { + bat """ + git rm -rf . + git clean -fxd + git reset --hard + """ + } + } + } + def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) @@ -236,6 +255,9 @@ def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repo setlfsfetchinclude(targetDir, sparseitemlist) } docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitsparserestore(targetDir) + } } def checkRepoExists(String gitURL) { From 35ef539f5c85632660016fcbd60a58179001894b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 May 2019 00:16:04 +0100 Subject: [PATCH 043/177] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ccbd458..8886076 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -232,15 +232,15 @@ def gitsparserestore(String targetDir) { dir(targetDir) { if (isUnix()) { sh """ - git rm -rf . - git clean -fxd - git reset --hard + git rm -rf . > /dev/null + git clean -fxd > /dev/null + git reset --hard > /dev/null """ } else { bat """ - git rm -rf . - git clean -fxd - git reset --hard + git rm -rf . 2> nul + git clean -fxd 2> nul + git reset --hard 2> nul """ } } From c1ceeca1430b9d90f792572c38291bbc7ea60aab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 May 2019 00:22:01 +0100 Subject: [PATCH 044/177] Try to get sparse checkout to work with lfs --- publish/publish.groovy | 9 --------- 1 file changed, 9 deletions(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index e4539f1..b478a5d 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -32,15 +32,6 @@ pipeline { } } } - stage('Test sparse checkout') { - steps { - script { - def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def ArrayList publishablepaths = ['installers/KM/*'] - gh.gitCheckoutAlt('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone - } - } - } stage('Get release artifact meta') { steps { script { From 26650ee254403cc2d6a850be1241b9000e124530 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:27:20 +0100 Subject: [PATCH 045/177] Add skeletal meta differences script files --- differences/dometahierarchy.groovy | 10 ++++ differences/metahierarchy.groovy | 92 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 differences/dometahierarchy.groovy create mode 100644 differences/metahierarchy.groovy diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy new file mode 100644 index 0000000..65e6cf4 --- /dev/null +++ b/differences/dometahierarchy.groovy @@ -0,0 +1,10 @@ + +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String summaryfolder, int maxdepth) { + def String summarystring = "No summary" + def boolean markunstable = true + def unstablereason = "No compare implementation yet" + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy new file mode 100644 index 0000000..3ea9dbc --- /dev/null +++ b/differences/metahierarchy.groovy @@ -0,0 +1,92 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + SUMMARYDIR = "${WORKSPACE}/summary" + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + "${WORKSPACE}/early", + "${WORKSPACE}/later", + "${SUMMARYDIR}", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout top level meta') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = 'git@hydra:gis/newsw.git' + + EARLYREVISION = "${RELEASE_EARLY_REVISION}".trim() + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) + + LATERREVISION = "${RELEASE_LATER_REVISION}".trim() + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) + } + } + } + stage('Diff components') { + steps { + script { + def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, SUMMARYDIR, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From f70406e72ba6ed5188134541454dd78a8f9561e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:35:45 +0100 Subject: [PATCH 046/177] Add skeletal meta differences script files --- differences/metahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 3ea9dbc..d82fcc0 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -29,14 +29,14 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - SOURCEURL = 'git@hydra:gis/newsw.git' + SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - EARLYREVISION = "${RELEASE_EARLY_REVISION}".trim() + EARLYREVISION = RELEASE_EARLY_REVISION.trim() EARLYROOTDIR = 'early' EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) - LATERREVISION = "${RELEASE_LATER_REVISION}".trim() + LATERREVISION = RELEASE_LATER_REVISION.trim() LATERROOTDIR = 'later' LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) From 36224b7c865a8218da05cc410ad96e2eddf50b86 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:48:25 +0100 Subject: [PATCH 047/177] Add skeletal meta differences script files --- differences/dometahierarchy.groovy | 6 +++++- differences/metahierarchy.groovy | 2 +- release/do.groovy | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 65e6cf4..1d6885f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,5 +1,9 @@ -def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String summaryfolder, int maxdepth) { +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def int md = maxdepth.toInteger() + + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index d82fcc0..cd6738d 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -47,7 +47,7 @@ pipeline { steps { script { def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") - (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, SUMMARYDIR, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' diff --git a/release/do.groovy b/release/do.groovy index b3d54c7..738c04a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -365,7 +365,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri } def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { - int md = maxdepth.toInteger() + def int md = maxdepth.toInteger() resetfolders() preconditions() From 7e5c624d72bec8116849ac59e57cb75272981407 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 May 2019 11:02:28 +0100 Subject: [PATCH 048/177] Strip *@tmp folders from published folder --- versioncontrol/git.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8886076..534084d 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -323,6 +323,7 @@ def stripgithistory(String rootfolder) { find . -name ".gitignore" | xargs -iXX rm -rf XX find . -name ".gitmodules" | xargs -iXX rm -rf XX find . -iname ".gitattributes" | xargs -iXX rm -rf XX + find . -iname "*@tmp" | xargs -iXX rm -rf XX """ } else { echo "Not supporting git stripping in Windows at the moment" From 8961f250b17a89620fa0ea35b796d664512fdfb8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 19:39:52 +0100 Subject: [PATCH 049/177] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 23 ++++++++++++++++++++++- publish/do.groovy | 13 +++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 34707bc..9620e8a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,7 +42,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -76,6 +76,27 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(destpath) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..6207380 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,6 +148,19 @@ def convertitemstopublish(String itemstopublishstring) { return paths } +def convertstringitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { From e418c593be9bb8ba865142321491b846567c3a7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:04:39 +0100 Subject: [PATCH 050/177] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 9620e8a..d85818f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) From a5a2603e27dfd1e94c84b134a06d07d0dfc5f8f6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:25:53 +0100 Subject: [PATCH 051/177] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- publish/do.groovy | 13 ------------- versioncontrol/git.groovy | 13 +++++++++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index d85818f..764618c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) diff --git a/publish/do.groovy b/publish/do.groovy index 6207380..1599ca8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,19 +148,6 @@ def convertitemstopublish(String itemstopublishstring) { return paths } -def convertstringitemstopublish(String itemstopublishstring) { - def List paths = [] - - def String[] items = itemstopublishstring.split(',') - def String allpaths = "" - for(String item : items) { - paths.add(item) - allpaths += "${path}\n" - } - echo "Selected item to publish:\n${allpaths}" - return paths -} - // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 534084d..818734a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -246,6 +246,19 @@ def gitsparserestore(String targetDir) { } } +def List convertSparseItemsString(String itemstopublish) { + def List paths = [] + + def String[] items = itemstopublish.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) From f1cb5075a2794e74a09d58c5f50a118f9813eeff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:33:55 +0100 Subject: [PATCH 052/177] MCS-425 - Perform a sparse checkout of items to import from a release repo --- versioncontrol/git.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 818734a..811d072 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -247,16 +247,16 @@ def gitsparserestore(String targetDir) { } def List convertSparseItemsString(String itemstopublish) { - def List paths = [] + def List sparsepaths = [] def String[] items = itemstopublish.split(',') def String allpaths = "" for(String item : items) { - paths.add(item) - allpaths += "${path}\n" + sparsepaths.add(item) + allpaths += "${item}\n" } echo "Selected item to publish:\n${allpaths}" - return paths + return sparsepaths } def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { From 2c5caba8ec8d123c2e4f852a7c33baafce6723e2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 13:30:15 +0100 Subject: [PATCH 053/177] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- meta/folder.groovy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 764618c..fd43379 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -68,7 +68,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) diff --git a/meta/folder.groovy b/meta/folder.groovy index b16db5d..f8c033b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -491,6 +491,7 @@ class MetaInfo extends GISJsonSerialised { } void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") releaseinfo.markreleased() for (associated in associatedmeta) { associated.meta.releaseinfo.markreleased() From 633a8dc8cb9bd107300c238a539cf97a56b5bcec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 14:08:13 +0100 Subject: [PATCH 054/177] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index fd43379..09eb337 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -89,7 +89,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) From b1d34d8f8795d1c6629537a7a5a5bc9b911becb4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 28 Jun 2019 18:25:28 +0100 Subject: [PATCH 055/177] Make the default filter blank; Alter the order of build import --- import/importer.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 09eb337..418fd9a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -17,7 +17,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, bo return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -42,7 +42,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -76,7 +76,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d return importedmeta } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -97,7 +97,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) From 2907eb9dc5ed4ff20295a5ca4af4d816b1506553 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 09:54:24 +0100 Subject: [PATCH 056/177] Make the default filter blank; Alter the order of build import --- versioncontrol/git.groovy | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 811d072..7e065d5 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -249,13 +249,16 @@ def gitsparserestore(String targetDir) { def List convertSparseItemsString(String itemstopublish) { def List sparsepaths = [] - def String[] items = itemstopublish.split(',') - def String allpaths = "" - for(String item : items) { - sparsepaths.add(item) - allpaths += "${item}\n" + boolean nonempty = itemstopublish?.trim() + if (nonempty) { + def String[] items = itemstopublish..trim().split(',') + def String allpaths = "" + for(String item : items) { + sparsepaths.add(item) + allpaths += "${item}\n" + } + echo "Selected item to publish:\n${allpaths}" } - echo "Selected item to publish:\n${allpaths}" return sparsepaths } From 7577060104098b6daf10a463b7758ce9d29930fd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 10:49:46 +0100 Subject: [PATCH 057/177] Remove redundant references to temp folders --- publish/do.groovy | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..18ec404 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -8,7 +8,6 @@ summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" -tempfolder = "${WORKSPACE}/temp" def resetfolders() { dir(summaryfolder) { @@ -20,9 +19,6 @@ def resetfolders() { dir(filteredreleaserepofolder) { deleteDir() } - dir(tempfolder) { - deleteDir() - } } def productuniquefolder(releasemeta) { @@ -61,8 +57,7 @@ def guessreporootsuffix(componentURL) { } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { - // Copy relevant artifacts to temp folder +def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { summaryappend("Artifacts: ") def releasemeta = releaseinfo.meta @@ -124,11 +119,10 @@ def copydirectory(File dirFrom, File dirTo) { def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) - def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) summaryappend("\n") } From 575fade931e0c0e1e83adc7168c44cef606015d2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 12:59:46 +0100 Subject: [PATCH 058/177] Add an option to import from repo to import lower levels of artifacts --- import/importer.groovy | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 418fd9a..68b8e4c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,11 +42,14 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } +publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - gh.gitCheckout(rooturl, rootrevision, destpath) + + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) @@ -59,8 +62,11 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckout(cloneurl, clonerevision, clonepath) + + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" echo "Marking related meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(relatedmeta, metafilepath) @@ -90,18 +96,21 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean recursive = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) case "Release repository": + if (nested) { + return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) default: error("Unknown type of artifact source: ${source}") From d9cd1acbb8fc5196ecb51dfe2494653ec297fa87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 13:19:05 +0100 Subject: [PATCH 059/177] Add an option to import from repo to import lower levels of artifacts --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 68b8e4c..ba5eb0b 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -103,7 +103,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean recursive = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) From f209f2a9d824ac0a4aac5da2f76af397d8237951 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Jul 2019 08:51:06 +0100 Subject: [PATCH 060/177] Further meta hierarchy difference work --- configuration/configuration.groovy | 2 +- differences/dometahierarchy.groovy | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 30b7086..1864ba9 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -9,7 +9,7 @@ def registerPipelineScriptConfiguration(meta) { } def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { - gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1d6885f..f5de8e8 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,3 +1,12 @@ +metafolderscript = load("${commonscripts()}/meta/folder.groovy") + +def showmetaitems(metaitems) + for (entry in metaitems) { + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath + echo "Name: ${name}\t\t\t\tPath: ${offsetpath}" + } +} def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def int md = maxdepth.toInteger() @@ -7,6 +16,18 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" + + def earlymeta = metafolderscript.importrelated(earlymetafolder) + def latermeta = metafolderscript.importrelated(latermetafolder) + + def earlymetaitems = earlymeta.releasewalkerunique(md) + def latermetaitems = latermeta.releasewalkerunique(md) + + echo "Early items:" + showmetaitems(earlymetaitems) + + echo "Later items:" + showmetaitems(latermetaitems) return [summarystring, markunstable, unstablereason] } From d68196c2c62eb7ae0c7c52783c6a726a06625cea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Jul 2019 13:26:33 +0100 Subject: [PATCH 061/177] Further meta hierarchy difference work --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f5de8e8..fa11a2d 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,6 +1,6 @@ metafolderscript = load("${commonscripts()}/meta/folder.groovy") -def showmetaitems(metaitems) +def showmetaitems(metaitems) { for (entry in metaitems) { def name = entry.meta.componentversionuniqueidentifier() def offsetpath = entry.offsetpath From 6cfd67289696fbc49560a91c2bc9f860f2402bca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Jul 2019 13:32:54 +0100 Subject: [PATCH 062/177] Further meta hierarchy difference work --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index fa11a2d..7f60aae 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,4 +1,4 @@ -metafolderscript = load("${commonscripts()}/meta/folder.groovy") +metafolderscript = load("${autobuilder.commonscripts()}/meta/folder.groovy") def showmetaitems(metaitems) { for (entry in metaitems) { From ebcc588604e682ec6296fba51b312ab670ecfb3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 08:05:43 +0100 Subject: [PATCH 063/177] No need to import meta script - attempting to remove jenkins crash --- differences/dometahierarchy.groovy | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 7f60aae..b6c9660 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,5 +1,3 @@ -metafolderscript = load("${autobuilder.commonscripts()}/meta/folder.groovy") - def showmetaitems(metaitems) { for (entry in metaitems) { def name = entry.meta.componentversionuniqueidentifier() @@ -11,18 +9,18 @@ def showmetaitems(metaitems) { def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def int md = maxdepth.toInteger() - echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" - - def earlymeta = metafolderscript.importrelated(earlymetafolder) - def latermeta = metafolderscript.importrelated(latermetafolder) - + + def earlymeta = meta.importrelated(earlymetafolder) + def latermeta = meta.importrelated(latermetafolder) + def earlymetaitems = earlymeta.releasewalkerunique(md) def latermetaitems = latermeta.releasewalkerunique(md) - + echo "Early items:" showmetaitems(earlymetaitems) From 8b4fbe2005f4a53731e8646d4a11cee00263a3ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 08:26:53 +0100 Subject: [PATCH 064/177] No need to import meta script - attempting to remove jenkins crash --- differences/dometahierarchy.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index b6c9660..62cd5ee 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,8 +1,8 @@ def showmetaitems(metaitems) { - for (entry in metaitems) { - def name = entry.meta.componentversionuniqueidentifier() - def offsetpath = entry.offsetpath - echo "Name: ${name}\t\t\t\tPath: ${offsetpath}" + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + def offsetpath = relatedmetainfo.offsetpath + echo "Name: ${key}\t\t\t\tPath: ${offsetpath}" } } From 01ff6c1a5f46f883662e1268b925c03f9d8a6b94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 08:47:33 +0100 Subject: [PATCH 065/177] No need to import meta script - attempting to remove jenkins crash --- differences/dometahierarchy.groovy | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 62cd5ee..1a6ed7f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,9 +1,23 @@ def showmetaitems(metaitems) { + String info = '' for (key in metaitems.keySet()) { def relatedmetainfo = metaitems[key] def offsetpath = relatedmetainfo.offsetpath - echo "Name: ${key}\t\t\t\tPath: ${offsetpath}" + info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" } + echo info +} + +def compare(earlymetaitems, latermetaitems, key) { + echo "Collecting changes between early and later versions of component: ${key}" +} + +def deletedcomponent(earlymetaitems, key) { + echo "Component ${key} has been deleted in later version" +} + +def addedcomponent(latermetaitems, key) { + echo "Component ${key} has been added in later version" } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { @@ -26,6 +40,19 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Later items:" showmetaitems(latermetaitems) + + for (key in earlymetaitems.keySet()) { + if (latermetaitems.containsKey(key)) { + compare(earlymetaitems, latermetaitems, key) + } else { + deletedcomponent(earlymetaitems, key) + } + } + for (key in latermetaitems.keySet()) { + if (!earlymetaitems.containsKey(key)) { + addedcomponent(latermetaitems, key) + } + } return [summarystring, markunstable, unstablereason] } From 22a5b7bde7460474ac9e1c9f148b837bac82bd0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 15:17:21 +0100 Subject: [PATCH 066/177] More infrastructure for differencing versions of component --- differences/do.groovy | 20 ++++++++ differences/dometahierarchy.groovy | 76 ++++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 32f18b5..1499e6d 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,4 +15,24 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def uniquefolderforsourcerepo = getcomponent(earlymeta) + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + } else { + echo "The early and later git urls don't match" + result = false; + } + + return [result, differencerecords] +} + return this; diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1a6ed7f..6ca7e08 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,3 +1,44 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" + +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + def showmetaitems(metaitems) { String info = '' for (key in metaitems.keySet()) { @@ -8,8 +49,33 @@ def showmetaitems(metaitems) { echo info } -def compare(earlymetaitems, latermetaitems, key) { +def String getcomponent(meta) { + def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + summaryappend("${meta.productinfo.productname} ${meta.productinfo.subcomponentname}: ") + + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + releaser.checkoutsourcerepo(meta, uniquefolderforsourcerepo) + + summaryappend("\n") + return uniquefolderforsourcerepo +} + +def boolean compare(earlymetaitems, latermetaitems, key) { + boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].meta + def latermeta = latermetaitems[key].meta + + def uniquefolderforsourcerepo = getcomponent(earlymeta) + + def differencer = load("${autobuilder.commonscripts()}/differences/do.groovy") + (result, differencerecords) = differencer.getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + return result; } def deletedcomponent(earlymetaitems, key) { @@ -22,12 +88,14 @@ def addedcomponent(latermetaitems, key) { def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def int md = maxdepth.toInteger() + resetfolders() + preconditions() echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - def String summarystring = "No summary" - def boolean markunstable = true - def unstablereason = "No compare implementation yet" + summarystring = "No summary" + markunstable = true + unstablereason = "No compare implementation yet" def earlymeta = meta.importrelated(earlymetafolder) def latermeta = meta.importrelated(latermetafolder) From 4dfde882fea490a115158a8ecb2b72508bb1b66a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 15:28:24 +0100 Subject: [PATCH 067/177] More infrastructure for differencing versions of component --- differences/dometahierarchy.groovy | 3 --- 1 file changed, 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6ca7e08..5047ce2 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -12,9 +12,6 @@ def resetfolders() { dir(summaryfolder) { deleteDir() } - dir(releaserepofolder) { - deleteDir() - } dir(tempfolder) { deleteDir() } From cc262c09ad51cedc1908203b99b760e65c9f5767 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 15:45:04 +0100 Subject: [PATCH 068/177] More infrastructure for differencing versions of component --- differences/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/differences/do.groovy b/differences/do.groovy index 1499e6d..6c5c947 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -23,7 +23,6 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def latergitinfo = latermeta.gitinfo if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { - def uniquefolderforsourcerepo = getcomponent(earlymeta) def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" From 437e27b0e2e67b31e1c03a28ba2c30a0741d8f7b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 16:09:14 +0100 Subject: [PATCH 069/177] More infrastructure for differencing versions of component --- differences/do.groovy | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/differences/do.groovy b/differences/do.groovy index 6c5c947..0bf3f4f 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,6 +15,20 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def differencerecords = [] + def String releaseexpression + if tripledot: + releaseexpression = earlyrevision + "..." + laterrevision + else: + releaseexpression = earlyrevision + ".." + laterrevision + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() + } + return differencetext +} + def getdifferencerecords(earlymeta, latermeta, String sourcepath) { boolean result = true; def List differencerecords = [] @@ -26,6 +40,8 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + echo "Difference text is:\n${differencetext}" } else { echo "The early and later git urls don't match" result = false; From 9458a9485a8be66e046e562974db7c3865d453ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 16:12:44 +0100 Subject: [PATCH 070/177] More infrastructure for differencing versions of component --- differences/do.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 0bf3f4f..eb4a53a 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -18,10 +18,11 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { def differencerecords = [] def String releaseexpression - if tripledot: + if (tripledot) { releaseexpression = earlyrevision + "..." + laterrevision - else: + } else { releaseexpression = earlyrevision + ".." + laterrevision + } def String differencetext dir(gitrootfolder) { differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() From 79405e223b00fb298bbb011abef8687e11058277 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:04:24 +0100 Subject: [PATCH 071/177] More infrastructure for differencing versions of component --- differences/do.groovy | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/differences/do.groovy b/differences/do.groovy index eb4a53a..99596f3 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,6 +15,20 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } +class DifferenceRecord { + String productname + String subcomponentname + String jirakey + + String rawdifferencetext + String sha + int type + String jiraticketinfo + + DifferenceRecord() { + } +} + def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { def differencerecords = [] def String releaseexpression @@ -30,6 +44,26 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late return differencetext } +def List processtext(meta, String text) { + def List differencerecords = [] + + text.eachLine { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = it.trim() + dr.sha = "sha" + dr.type = 0 + dr.jiraticketinfo = "info" + + differencerecords.add(dr) + } + return differencerecords +} + def getdifferencerecords(earlymeta, latermeta, String sourcepath) { boolean result = true; def List differencerecords = [] @@ -43,6 +77,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) echo "Difference text is:\n${differencetext}" + differencerecords = differencerecords + processtext(earlymeta, differencetext) } else { echo "The early and later git urls don't match" result = false; From d202e0bb3e4460735385f25a1f32bdb45cac9970 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:19:12 +0100 Subject: [PATCH 072/177] More infrastructure for differencing versions of component --- differences/do.groovy | 11 +++++++++-- differences/dometahierarchy.groovy | 13 +++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 99596f3..b29aaa4 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -16,6 +16,8 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } class DifferenceRecord { + def script + String productname String subcomponentname String jirakey @@ -25,7 +27,12 @@ class DifferenceRecord { int type String jiraticketinfo - DifferenceRecord() { + DifferenceRecord(script) { + this.script = script + } + + def dump() { + script.echo("Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" } } @@ -48,7 +55,7 @@ def List processtext(meta, String text) { def List differencerecords = [] text.eachLine { - def DifferenceRecord dr = new DifferenceRecord() + def DifferenceRecord dr = new DifferenceRecord(this) dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5047ce2..464dce9 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -60,7 +60,7 @@ def String getcomponent(meta) { return uniquefolderforsourcerepo } -def boolean compare(earlymetaitems, latermetaitems, key) { +def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" @@ -72,7 +72,7 @@ def boolean compare(earlymetaitems, latermetaitems, key) { def differencer = load("${autobuilder.commonscripts()}/differences/do.groovy") (result, differencerecords) = differencer.getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - return result; + return [result, differencerecords]; } def deletedcomponent(earlymetaitems, key) { @@ -84,6 +84,8 @@ def addedcomponent(latermetaitems, key) { } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def List differencerecords = [] + def int md = maxdepth.toInteger() resetfolders() preconditions() @@ -108,7 +110,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { - compare(earlymetaitems, latermetaitems, key) + (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + differencerecords = differencerecords + componentdifferencerecords } else { deletedcomponent(earlymetaitems, key) } @@ -118,7 +121,9 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin addedcomponent(latermetaitems, key) } } - + for (record in differencerecords) { + record.dump() + } return [summarystring, markunstable, unstablereason] } From 28d24268fb9a78a98ac492092fe983d1a40f99ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:29:03 +0100 Subject: [PATCH 073/177] Add more difference infrastructure --- differences/do.groovy | 6 ++---- differences/dometahierarchy.groovy | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index b29aaa4..0de5395 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -16,8 +16,6 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } class DifferenceRecord { - def script - String productname String subcomponentname String jirakey @@ -31,8 +29,8 @@ class DifferenceRecord { this.script = script } - def dump() { - script.echo("Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" + def String dump() { + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" } } diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 464dce9..2770c4a 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -122,7 +122,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin } } for (record in differencerecords) { - record.dump() + echo "${record.dump()}" } return [summarystring, markunstable, unstablereason] } From f00082dd8b3fa3daf6e65669cb48d4fba4e99076 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:51:01 +0100 Subject: [PATCH 074/177] Add more difference infrastructure --- differences/do.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 0de5395..611ce68 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -25,8 +25,7 @@ class DifferenceRecord { int type String jiraticketinfo - DifferenceRecord(script) { - this.script = script + DifferenceRecord() { } def String dump() { @@ -53,7 +52,7 @@ def List processtext(meta, String text) { def List differencerecords = [] text.eachLine { - def DifferenceRecord dr = new DifferenceRecord(this) + def DifferenceRecord dr = new DifferenceRecord() dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname From 02b104312dceca59b681c8911501b3593c070888 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 19:34:26 +0100 Subject: [PATCH 075/177] Add more difference infrastructure --- differences/do.groovy | 75 -------------------- differences/dometahierarchy.groovy | 106 +++++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 79 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 611ce68..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,79 +15,4 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } -class DifferenceRecord { - String productname - String subcomponentname - String jirakey - - String rawdifferencetext - String sha - int type - String jiraticketinfo - - DifferenceRecord() { - } - - def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" - } -} - -def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { - def differencerecords = [] - def String releaseexpression - if (tripledot) { - releaseexpression = earlyrevision + "..." + laterrevision - } else { - releaseexpression = earlyrevision + ".." + laterrevision - } - def String differencetext - dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() - } - return differencetext -} - -def List processtext(meta, String text) { - def List differencerecords = [] - - text.eachLine { - def DifferenceRecord dr = new DifferenceRecord() - - dr.productname = meta.productinfo.productname - dr.subcomponentname = meta.productinfo.subcomponentname - dr.jirakey = meta.productinfo.jirakey - - dr.rawdifferencetext = it.trim() - dr.sha = "sha" - dr.type = 0 - dr.jiraticketinfo = "info" - - differencerecords.add(dr) - } - return differencerecords -} - -def getdifferencerecords(earlymeta, latermeta, String sourcepath) { - boolean result = true; - def List differencerecords = [] - - def earlygitinfo = earlymeta.gitinfo - def latergitinfo = latermeta.gitinfo - - if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { - def earlyrevision = earlygitinfo.gitrevision - def laterrevision = latergitinfo.gitrevision - echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" - def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - echo "Difference text is:\n${differencetext}" - differencerecords = differencerecords + processtext(earlymeta, differencetext) - } else { - echo "The early and later git urls don't match" - result = false; - } - - return [result, differencerecords] -} - return this; diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 2770c4a..f60efd7 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -8,6 +8,24 @@ summarystring = "" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" +class DifferenceRecord { + String productname + String subcomponentname + String jirakey + + String rawdifferencetext + String sha + int type + String jiraticketinfo + + DifferenceRecord() { + } + + def String dump() { + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" + } +} + def resetfolders() { dir(summaryfolder) { deleteDir() @@ -46,6 +64,31 @@ def showmetaitems(metaitems) { echo info } +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } +} + def String getcomponent(meta) { def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -53,13 +96,69 @@ def String getcomponent(meta) { summaryappend("${meta.productinfo.productname} ${meta.productinfo.subcomponentname}: ") - def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") - releaser.checkoutsourcerepo(meta, uniquefolderforsourcerepo) + checkoutsourcerepo(meta, uniquefolderforsourcerepo) summaryappend("\n") return uniquefolderforsourcerepo } +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def differencerecords = [] + def String releaseexpression + if (tripledot) { + releaseexpression = earlyrevision + "..." + laterrevision + } else { + releaseexpression = earlyrevision + ".." + laterrevision + } + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() + } + return differencetext +} + +def List processtext(meta, String text) { + def List differencerecords = [] + + text.eachLine { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = it.trim() + dr.sha = "sha" + dr.type = 0 + dr.jiraticketinfo = "info" + + differencerecords.add(dr) + } + return differencerecords +} + +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + echo "Difference text is:\n${differencetext}" + differencerecords = processtext(earlymeta, differencetext) + } else { + echo "The early and later git urls don't match" + result = false; + } + + return [result, differencerecords] +} + def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" @@ -69,8 +168,7 @@ def compare(earlymetaitems, latermetaitems, key) { def uniquefolderforsourcerepo = getcomponent(earlymeta) - def differencer = load("${autobuilder.commonscripts()}/differences/do.groovy") - (result, differencerecords) = differencer.getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) return [result, differencerecords]; } From 665e0039adb0f3a5a64efd0f77a3f9e763f9d523 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 19:52:53 +0100 Subject: [PATCH 076/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f60efd7..9e166b7 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -120,9 +120,11 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def List processtext(meta, String text) { def List differencerecords = [] + record = 0; text.eachLine { def DifferenceRecord dr = new DifferenceRecord() + echo "Creating record: ${record}" dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname dr.jirakey = meta.productinfo.jirakey @@ -133,6 +135,7 @@ def List processtext(meta, String text) { dr.jiraticketinfo = "info" differencerecords.add(dr) + record += 1 } return differencerecords } @@ -169,7 +172,7 @@ def compare(earlymetaitems, latermetaitems, key) { def uniquefolderforsourcerepo = getcomponent(earlymeta) (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - + return [result, differencerecords]; } @@ -209,6 +212,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + echo "Returned: ${result}, records: ${componentdifferencerecords.size()}" + differencerecords = differencerecords + componentdifferencerecords } else { deletedcomponent(earlymetaitems, key) @@ -219,6 +224,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin addedcomponent(latermetaitems, key) } } + echo "Result: ${result}, total records: ${differencerecords.size()}" for (record in differencerecords) { echo "${record.dump()}" } From c75e58cc697dac6bed9d4bcaf4ba7277a0b30d72 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 19:59:31 +0100 Subject: [PATCH 077/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9e166b7..97efa08 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -121,7 +121,7 @@ def List processtext(meta, String text) { def List differencerecords = [] record = 0; - text.eachLine { + text.split('\n').each { def DifferenceRecord dr = new DifferenceRecord() echo "Creating record: ${record}" From fe72ba4a9f1b4e210508f2afff1cd3e18c86d60f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 08:40:22 +0100 Subject: [PATCH 078/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 97efa08..dc35bdf 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -94,11 +94,8 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - summaryappend("${meta.productinfo.productname} ${meta.productinfo.subcomponentname}: ") - checkoutsourcerepo(meta, uniquefolderforsourcerepo) - summaryappend("\n") return uniquefolderforsourcerepo } @@ -120,11 +117,9 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def List processtext(meta, String text) { def List differencerecords = [] - record = 0; text.split('\n').each { def DifferenceRecord dr = new DifferenceRecord() - echo "Creating record: ${record}" dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname dr.jirakey = meta.productinfo.jirakey @@ -135,7 +130,6 @@ def List processtext(meta, String text) { dr.jiraticketinfo = "info" differencerecords.add(dr) - record += 1 } return differencerecords } @@ -151,12 +145,14 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - echo "Difference text is:\n${differencetext}" differencerecords = processtext(earlymeta, differencetext) } else { + summaryappend("The early and later git urls don't match") echo "The early and later git urls don't match" result = false; + setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") } return [result, differencerecords] @@ -165,6 +161,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" + summaryappend("Changed: ") def earlymeta = earlymetaitems[key].meta def latermeta = latermetaitems[key].meta @@ -178,10 +175,12 @@ def compare(earlymetaitems, latermetaitems, key) { def deletedcomponent(earlymetaitems, key) { echo "Component ${key} has been deleted in later version" + summaryappend("Deleted ") } def addedcomponent(latermetaitems, key) { echo "Component ${key} has been added in later version" + summaryappend("Added ") } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { @@ -193,10 +192,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - summarystring = "No summary" - markunstable = true - unstablereason = "No compare implementation yet" - def earlymeta = meta.importrelated(earlymetafolder) def latermeta = meta.importrelated(latermetafolder) @@ -210,21 +205,26 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin showmetaitems(latermetaitems) for (key in earlymetaitems.keySet()) { + def earlymeta = earlymetaitems[key].meta if (latermetaitems.containsKey(key)) { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) - echo "Returned: ${result}, records: ${componentdifferencerecords.size()}" - differencerecords = differencerecords + componentdifferencerecords + summaryappend("\n") } else { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) + summaryappend("\n") } } for (key in latermetaitems.keySet()) { + def latermeta = latermetaitems[key].meta if (!earlymetaitems.containsKey(key)) { + summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) + summaryappend("\n") } } - echo "Result: ${result}, total records: ${differencerecords.size()}" for (record in differencerecords) { echo "${record.dump()}" } From 319ae116ca3af419130919ce71b136f43b1d5af4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 09:06:52 +0100 Subject: [PATCH 079/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dc35bdf..0b42152 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -205,7 +205,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin showmetaitems(latermetaitems) for (key in earlymetaitems.keySet()) { - def earlymeta = earlymetaitems[key].meta if (latermetaitems.containsKey(key)) { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) @@ -218,7 +217,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin } } for (key in latermetaitems.keySet()) { - def latermeta = latermetaitems[key].meta if (!earlymetaitems.containsKey(key)) { summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) From d1d69c6c63c9f2aa9cd7bd8d996264dab1828804 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:03:16 +0100 Subject: [PATCH 080/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 45 ++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 0b42152..bbfde2b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -114,21 +114,42 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late return differencetext } +def extractsha(String line) { + def pattern = ~"^(.*)\s(.*)" + + def sha = line.replaceFirst(pattern) { _, sha, therest -> "${sha.trim()}" } + def therest = line.replaceFirst(pattern) { _, sha, therest -> "${therest.trim()}" } + + echo "Sha filtering: line: ${line} sha: ${sha} therest: ${therest}" + + return [sha, therest] +} + +def DifferenceRecord fillrecord(meta, line) { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = line + + // Example of raw text to process: + // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py + + (dr.sha, line) = extractsha(line) + + dr.type = 0 + dr.jiraticketinfo = "info" + + return dr +} + def List processtext(meta, String text) { def List differencerecords = [] - - text.split('\n').each { - def DifferenceRecord dr = new DifferenceRecord() - - dr.productname = meta.productinfo.productname - dr.subcomponentname = meta.productinfo.subcomponentname - dr.jirakey = meta.productinfo.jirakey - dr.rawdifferencetext = it.trim() - dr.sha = "sha" - dr.type = 0 - dr.jiraticketinfo = "info" - + text.split('\n').each { + DifferenceRecord dr = fillrecord(meta, it.trim()) differencerecords.add(dr) } return differencerecords From a08335c68eb0712f9266182b84d15e6a9a05754b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:07:47 +0100 Subject: [PATCH 081/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index bbfde2b..395b478 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -115,7 +115,7 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late } def extractsha(String line) { - def pattern = ~"^(.*)\s(.*)" + def pattern = ~"^(.*)\\s(.*)" def sha = line.replaceFirst(pattern) { _, sha, therest -> "${sha.trim()}" } def therest = line.replaceFirst(pattern) { _, sha, therest -> "${therest.trim()}" } From a9ff43ff0aa8922e76f622218cdb18ddd579f35d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:14:53 +0100 Subject: [PATCH 082/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 395b478..912583d 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -137,7 +137,8 @@ def DifferenceRecord fillrecord(meta, line) { // Example of raw text to process: // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py - (dr.sha, line) = extractsha(line) + (sha, line) = extractsha(line) + dr.sha = sha dr.type = 0 dr.jiraticketinfo = "info" From a457787eeab03a0baa1cef505244f61357cec648 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:18:32 +0100 Subject: [PATCH 083/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 912583d..5350529 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -117,10 +117,10 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def extractsha(String line) { def pattern = ~"^(.*)\\s(.*)" - def sha = line.replaceFirst(pattern) { _, sha, therest -> "${sha.trim()}" } - def therest = line.replaceFirst(pattern) { _, sha, therest -> "${therest.trim()}" } + def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit.trim()}" } + def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest.trim()}" } - echo "Sha filtering: line: ${line} sha: ${sha} therest: ${therest}" + echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" return [sha, therest] } From ca296aac27be9194802b150a9ec9a3d6e15c9298 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 13:05:37 +0100 Subject: [PATCH 084/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5350529..f98b96f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -115,14 +115,14 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late } def extractsha(String line) { - def pattern = ~"^(.*)\\s(.*)" + def pattern = ~"^(\\p{XDigit}*) (.*)" - def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit.trim()}" } - def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest.trim()}" } + def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit}" } + def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest}" } echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" - return [sha, therest] + return [sha.trim(), therest.trim()] } def DifferenceRecord fillrecord(meta, line) { @@ -137,7 +137,7 @@ def DifferenceRecord fillrecord(meta, line) { // Example of raw text to process: // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py - (sha, line) = extractsha(line) + (sha, line2) = extractsha(line) dr.sha = sha dr.type = 0 From 90ed5663c64016860a6aba5883086a20bf6eef68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 13:08:43 +0100 Subject: [PATCH 085/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f98b96f..a58c02e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -122,7 +122,7 @@ def extractsha(String line) { echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" - return [sha.trim(), therest.trim()] + return [sha.trim(), restofline.trim()] } def DifferenceRecord fillrecord(meta, line) { From d51458d7b45d64a03305f1d30684f788d5a1e3e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 13:26:46 +0100 Subject: [PATCH 086/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a58c02e..583c7f1 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -116,9 +116,11 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def extractsha(String line) { def pattern = ~"^(\\p{XDigit}*) (.*)" + def match = (line =~ pattern) + match.find() - def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit}" } - def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest}" } + def sha = match.group(1) + def restofline = match.group(2) echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" From b51c69707a3a0de818238981d7a97ac0c830730b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 14:28:43 +0100 Subject: [PATCH 087/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 64 +++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 583c7f1..9e20da2 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -16,13 +16,15 @@ class DifferenceRecord { String rawdifferencetext String sha int type - String jiraticketinfo + String mergeinfo + List jiraticketinfo + String comment DifferenceRecord() { } def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: $(mergeinfo)\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" } } @@ -122,11 +124,45 @@ def extractsha(String line) { def sha = match.group(1) def restofline = match.group(2) - echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" + //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" return [sha.trim(), restofline.trim()] } +def extractmerges(String line) { + def pattern1 = "^(.*)(Merge branch .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + def pattern2 = "^(.*)(Merge remote-tracking branch .*)" + def match2 = (line =~ pattern2) + if (match2.find()) { + return [true, match2.group(2)] + } + + def pattern3 = "^(.*)(Merge commit .*)" + def match3 = (line =~ pattern3) + if (match3.find()) { + return [true, match3.group(2)] + } + + def pattern4 = "^(.*)(Add .* from commit .*)" + def match4 = (line =~ pattern4) + if (match4.find()) { + return [true, match4.group(2)] + } + + return [false, line] +} + +def extractjiraticketinfo(String line) { + bool result = true + + return [result, [], line] +} + def DifferenceRecord fillrecord(meta, line) { def DifferenceRecord dr = new DifferenceRecord() @@ -142,8 +178,26 @@ def DifferenceRecord fillrecord(meta, line) { (sha, line2) = extractsha(line) dr.sha = sha - dr.type = 0 - dr.jiraticketinfo = "info" + (result, mergeinfo) = extractmerges(line2) + if (result) { + dr.type = 1 + dr.jiraticketinfo = [] + dr.mergeinfo = mergeinfo + dr.comment = "" + } else { + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = 2 + dr.jiraticketinfo = jiraticketinfo + dr.mergeinfo = "" + dr.comment = comment + } else { + dr.type = 3 + dr.jiraticketinfo = [] + dr.mergeinfo = "" + df.comment = line2 + } + } return dr } From 299df9b6b219992d7760f9a68524545c9e57b72d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 14:31:16 +0100 Subject: [PATCH 088/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9e20da2..f997983 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -24,7 +24,7 @@ class DifferenceRecord { } def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: $(mergeinfo)\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" } } From 151bbc9599c1d8115d91f1220d0d449ed9c7322d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 18:27:49 +0100 Subject: [PATCH 089/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f997983..5b85543 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -158,7 +158,7 @@ def extractmerges(String line) { } def extractjiraticketinfo(String line) { - bool result = true + boolean result = true return [result, [], line] } From 1c5fae0ab028666c4b083da36ce822211bbfb542 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 19:09:23 +0100 Subject: [PATCH 090/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 47 +++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5b85543..bcc79dc 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -15,7 +15,7 @@ class DifferenceRecord { String rawdifferencetext String sha - int type + String type String mergeinfo List jiraticketinfo String comment @@ -163,6 +163,28 @@ def extractjiraticketinfo(String line) { return [result, [], line] } +def extractreverts(String line) { + boolean result = true + + def pattern1 = '^(.*)(Revert \"(.*)\"' + def match1 = (line =~ pattern1) + if (match1.find()) { + def line2 = match1.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + def pattern2 = '^(.*)(Partial Revert \"(.*)\"' + def match2 = (line =~ pattern2) + if (match2.find()) { + def line2 = match2.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + return [false, [], line] +} + def DifferenceRecord fillrecord(meta, line) { def DifferenceRecord dr = new DifferenceRecord() @@ -180,22 +202,25 @@ def DifferenceRecord fillrecord(meta, line) { (result, mergeinfo) = extractmerges(line2) if (result) { - dr.type = 1 - dr.jiraticketinfo = [] + dr.type = "Merge" dr.mergeinfo = mergeinfo - dr.comment = "" } else { - (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + (result, jiraticketinfo, comment) = extractreverts(line2) if (result) { - dr.type = 2 + dr.type = "Revert" dr.jiraticketinfo = jiraticketinfo - dr.mergeinfo = "" dr.comment = comment } else { - dr.type = 3 - dr.jiraticketinfo = [] - dr.mergeinfo = "" - df.comment = line2 + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.jiraticketinfo = [] + df.comment = line2 + } } } From a2a582cd3988a42fa7e8592079ee8a4f009464c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 20:44:51 +0100 Subject: [PATCH 091/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index bcc79dc..31b2e13 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -166,7 +166,7 @@ def extractjiraticketinfo(String line) { def extractreverts(String line) { boolean result = true - def pattern1 = '^(.*)(Revert \"(.*)\"' + def pattern1 = '^(.*)Revert \"(.*)\"' def match1 = (line =~ pattern1) if (match1.find()) { def line2 = match1.group(2) @@ -174,7 +174,7 @@ def extractreverts(String line) { return [true, originaljiraticketinfo, originalcomment] } - def pattern2 = '^(.*)(Partial Revert \"(.*)\"' + def pattern2 = '^(.*)Partial Revert \"(.*)\"' def match2 = (line =~ pattern2) if (match2.find()) { def line2 = match2.group(2) From c564a70e8de086b626aeaf65f2d8600639b3d354 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 22:08:04 +0100 Subject: [PATCH 092/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 46 ++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 31b2e13..6485c72 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -21,6 +21,16 @@ class DifferenceRecord { String comment DifferenceRecord() { + productname = "" + subcomponentname = "" + jirakey = "" + + rawdifferencetext = "" + sha = "" + type = "" + mergeinfo = "" + jiraticketinfo = [] + comment = "" } def String dump() { @@ -158,9 +168,40 @@ def extractmerges(String line) { } def extractjiraticketinfo(String line) { - boolean result = true + def List jiratickets = [] - return [result, [], line] + boolean amatch = false + while (!amatch) { + line = line.trim() + def pattern1 = '^(\w+-\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def match1 = (line =~ pattern1) + if (match1.find()) { + jiraticket = match1.group(1) + jiratickets.add(jiraticket) + line = match1.group(2) + amatch = true + } else { + def pattern2 = '^(\w+\d+)(.*)' //GIS1234 smb1234 + def match2 = (line =~ pattern2) + if (match2.find()) { + jiraticket = match2.group(1) + jiratickets.add(jiraticket) + line = match2.group(2) + amatch = true + } else { + def pattern3 = '^(\w+ \d+)(.*)' //GIS 1234 smb 1234 + def match3 = (line =~ pattern3) + if (match3.find()) { + jiraticket = match3.group(1) + jiratickets.add(jiraticket) + line = match3.group(2) + amatch = true + } + } + } + } + boolean result = jiratickets.size() > 0 + return [result, jiratickets, line] } def extractreverts(String line) { @@ -218,7 +259,6 @@ def DifferenceRecord fillrecord(meta, line) { dr.comment = comment } else { dr.type = "Unticketed" - dr.jiraticketinfo = [] df.comment = line2 } } From 1f6b1f8aae1032e2fd93c8e16760c1e440e31444 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 22:14:50 +0100 Subject: [PATCH 093/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6485c72..f3ffb80 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -173,7 +173,7 @@ def extractjiraticketinfo(String line) { boolean amatch = false while (!amatch) { line = line.trim() - def pattern1 = '^(\w+-\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) @@ -181,7 +181,7 @@ def extractjiraticketinfo(String line) { line = match1.group(2) amatch = true } else { - def pattern2 = '^(\w+\d+)(.*)' //GIS1234 smb1234 + def pattern2 = '^(\\w+\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) @@ -189,7 +189,7 @@ def extractjiraticketinfo(String line) { line = match2.group(2) amatch = true } else { - def pattern3 = '^(\w+ \d+)(.*)' //GIS 1234 smb 1234 + def pattern3 = '^(\\w+ \\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) From 6c0b5c63c2575104f907fce6a75e3464638fc7dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 23:59:08 +0100 Subject: [PATCH 094/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 65 +++++++++++++++++++++++++++++- differences/metahierarchy.groovy | 4 +- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f3ffb80..cc3d168 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -8,6 +8,8 @@ summarystring = "" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" +reportfolder = "${WORKSPACE}/reports" + class DifferenceRecord { String productname String subcomponentname @@ -45,6 +47,9 @@ def resetfolders() { dir(tempfolder) { deleteDir() } + dir(reportfolder) { + deleteDir() + } } def preconditions() { @@ -326,6 +331,62 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } +def boolean componentreport(meta, componentdifferencerecords) { + def ticketed = [:] + def merges = [:] + def reverts = [:] + def unticketed = [:] + def tickets = [:] + + for (record in componentdifferencerecords) { + if (dr.type == "Merge") { + merges[record.mergeinfo] = record + } + if (dr.type == "Revert") { + for (ticket in record.jiraticketinfo) { + if (!reverts.containsKey(ticket)) { + reverts[ticket] = [record.comment] + } else { + reverts[ticket].add(record.comment) + } + if (!tickets.containsKey(ticket)) { + tickets[ticket] = 1 + } else { + tickets[ticket] += 1 + } + } + } + if (dr.type == "Ticketed") { + for (ticket in record.jiraticketinfo) { + if (!ticketed.containsKey(ticket)) { + ticketed[ticket] = [record.comment] + } else { + ticketed[ticket].add(record.comment) + } + if (!tickets.containsKey(ticket)) { + tickets[ticket] = 1 + } else { + tickets[ticket] += 1 + } + } + } + if (dr.type == "Unticketed") { + unticketed[record.comment] = record + } + } + String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" + dir(componentreportfolder) { + String ticketlist = "Ticket list:\n" + for (ticketnumber in tickets.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key) { + ticketlist += "${ticketnumber} (${tickets[ticketnumber]})" + } + writeFile(file: "ticketlist", text: ticketlist) + } +} +m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key +tickets.sort { a, b -> a.value <=> b.value } + + def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] @@ -346,11 +407,13 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Later items:" showmetaitems(latermetaitems) - + for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + summaryappend("Items: ${componentdifferencerecords.size()} ") + componentreport(earlymeta, componentdifferencerecords) differencerecords = differencerecords + componentdifferencerecords summaryappend("\n") } else { diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index cd6738d..8a42f5e 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -13,11 +13,13 @@ pipeline { steps { script { SUMMARYDIR = "${WORKSPACE}/summary" + REPORTDIR = "${WORKSPACE}/reports" cleanfolders = [ "${WORKSPACE}/build_artifacts", "${WORKSPACE}/early", "${WORKSPACE}/later", "${SUMMARYDIR}", + "${REPORTDIR}", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -59,7 +61,7 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + archiveArtifacts "summary/**,reports/**,${artifactlist}" } } } From 943ecc2d8d9ea011d2c04a17487d85a735e840da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:02:02 +0100 Subject: [PATCH 095/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index cc3d168..65be091 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -383,8 +383,8 @@ def boolean componentreport(meta, componentdifferencerecords) { writeFile(file: "ticketlist", text: ticketlist) } } -m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key -tickets.sort { a, b -> a.value <=> b.value } +//m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key +//tickets.sort { a, b -> a.value <=> b.value } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { From 0a28daa24fa1ec6db4f6ddf33d98e864524aed65 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:04:38 +0100 Subject: [PATCH 096/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 65be091..d5f0fdb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -338,16 +338,16 @@ def boolean componentreport(meta, componentdifferencerecords) { def unticketed = [:] def tickets = [:] - for (record in componentdifferencerecords) { + for (dr in componentdifferencerecords) { if (dr.type == "Merge") { - merges[record.mergeinfo] = record + merges[dr.mergeinfo] = dr } if (dr.type == "Revert") { - for (ticket in record.jiraticketinfo) { + for (ticket in dr.jiraticketinfo) { if (!reverts.containsKey(ticket)) { - reverts[ticket] = [record.comment] + reverts[ticket] = [dr.comment] } else { - reverts[ticket].add(record.comment) + reverts[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 @@ -357,11 +357,11 @@ def boolean componentreport(meta, componentdifferencerecords) { } } if (dr.type == "Ticketed") { - for (ticket in record.jiraticketinfo) { + for (ticket in dr.jiraticketinfo) { if (!ticketed.containsKey(ticket)) { - ticketed[ticket] = [record.comment] + ticketed[ticket] = [dr.comment] } else { - ticketed[ticket].add(record.comment) + ticketed[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 @@ -371,7 +371,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } } if (dr.type == "Unticketed") { - unticketed[record.comment] = record + unticketed[dr.comment] = dr } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" From 99238fb9c521c89efeda2b2a0b0dd8fb4025f452 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:09:36 +0100 Subject: [PATCH 097/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index d5f0fdb..30a0442 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,7 +377,7 @@ def boolean componentreport(meta, componentdifferencerecords) { String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" dir(componentreportfolder) { String ticketlist = "Ticket list:\n" - for (ticketnumber in tickets.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key) { + for (ticketnumber in tickets.sort()*.key) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})" } writeFile(file: "ticketlist", text: ticketlist) @@ -385,7 +385,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key //tickets.sort { a, b -> a.value <=> b.value } - +//myMap.sort()*.key def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] From cfff9bd1025f9786ec6d3e580086f30de612785f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:32:12 +0100 Subject: [PATCH 098/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 45 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 30a0442..effa08f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -306,21 +306,6 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { return [result, differencerecords] } -def compare(earlymetaitems, latermetaitems, key) { - boolean result = true; - echo "Collecting changes between early and later versions of component: ${key}" - summaryappend("Changed: ") - - def earlymeta = earlymetaitems[key].meta - def latermeta = latermetaitems[key].meta - - def uniquefolderforsourcerepo = getcomponent(earlymeta) - - (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - - return [result, differencerecords]; -} - def deletedcomponent(earlymetaitems, key) { echo "Component ${key} has been deleted in later version" summaryappend("Deleted ") @@ -378,7 +363,7 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" for (ticketnumber in tickets.sort()*.key) { - ticketlist += "${ticketnumber} (${tickets[ticketnumber]})" + ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } writeFile(file: "ticketlist", text: ticketlist) } @@ -387,6 +372,26 @@ def boolean componentreport(meta, componentdifferencerecords) { //tickets.sort { a, b -> a.value <=> b.value } //myMap.sort()*.key +def compare(earlymetaitems, latermetaitems, key) { + boolean result = true; + echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].meta + def latermeta = latermetaitems[key].meta + + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + + def uniquefolderforsourcerepo = getcomponent(earlymeta) + + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + summaryappend(" Items: ${differencerecords.size()} ") + componentreport(earlymeta, differencerecords) + summaryappend("\n") + + return [result, differencerecords]; +} + def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] @@ -410,12 +415,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) - summaryappend("Items: ${componentdifferencerecords.size()} ") - componentreport(earlymeta, componentdifferencerecords) differencerecords = differencerecords + componentdifferencerecords - summaryappend("\n") } else { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) @@ -435,4 +436,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin return [summarystring, markunstable, unstablereason] } + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + + return this; From 7e3042e0d27b6334e590ec904a26b2eded64ac53 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:34:35 +0100 Subject: [PATCH 099/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index effa08f..f8650be 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -436,8 +436,4 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin return [summarystring, markunstable, unstablereason] } - for (key in metaitems.keySet()) { - def relatedmetainfo = metaitems[key] - - return this; From a260f34a1525cf6dd485af62e9c0be620222b203 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 08:21:49 +0100 Subject: [PATCH 100/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f8650be..5734bb6 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -324,38 +324,52 @@ def boolean componentreport(meta, componentdifferencerecords) { def tickets = [:] for (dr in componentdifferencerecords) { + echo "Processing:\n${dr.dump()}" + if (dr.type == "Merge") { + echo "Adding to mergeinfo" merges[dr.mergeinfo] = dr } if (dr.type == "Revert") { + echo "Adding to revert" for (ticket in dr.jiraticketinfo) { if (!reverts.containsKey(ticket)) { + echo "Creating reverts record for ${ticket}" reverts[ticket] = [dr.comment] } else { + echo "Adding to ticketed record for ${ticket}" reverts[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { + echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { + echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } } } if (dr.type == "Ticketed") { + echo "Adding to ticketed" for (ticket in dr.jiraticketinfo) { if (!ticketed.containsKey(ticket)) { + echo "Creating ticketed record for ${ticket}" ticketed[ticket] = [dr.comment] } else { + echo "Adding to ticketed record for ${ticket}" ticketed[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { + echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { + echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } } } if (dr.type == "Unticketed") { + echo "Adding to unticketed" unticketed[dr.comment] = dr } } @@ -430,9 +444,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin summaryappend("\n") } } - for (record in differencerecords) { - echo "${record.dump()}" - } return [summarystring, markunstable, unstablereason] } From 97faaf812e28ef0328d0d2863c0640d0cb3798f5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 08:31:05 +0100 Subject: [PATCH 101/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5734bb6..1b622e0 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -347,6 +347,7 @@ def boolean componentreport(meta, componentdifferencerecords) { echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } + echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Ticketed") { @@ -366,6 +367,7 @@ def boolean componentreport(meta, componentdifferencerecords) { echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } + echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Unticketed") { @@ -374,6 +376,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" + echo "Size of tickets map: ${tickets.size()}" dir(componentreportfolder) { String ticketlist = "Ticket list:\n" for (ticketnumber in tickets.sort()*.key) { From 9c2d045a1883be3a3dc5ac1687f5a1d789a763c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 08:36:05 +0100 Subject: [PATCH 102/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1b622e0..861dbca 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -379,6 +379,8 @@ def boolean componentreport(meta, componentdifferencerecords) { echo "Size of tickets map: ${tickets.size()}" dir(componentreportfolder) { String ticketlist = "Ticket list:\n" + echo "tickets map:\n${tickets}" + echo "tickets map sorted:\n${tickets.sort()*.key}" for (ticketnumber in tickets.sort()*.key) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } From 3936752cb5e47fadd0201dfa16abc85611af7fa6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 09:14:46 +0100 Subject: [PATCH 103/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 861dbca..1a4cb36 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -380,8 +380,9 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - echo "tickets map sorted:\n${tickets.sort()*.key}" - for (ticketnumber in tickets.sort()*.key) { + tickets = tickets.sort { it.key } + echo "tickets map sorted:\n${tickets}" + for (ticketnumber in tickets) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } writeFile(file: "ticketlist", text: ticketlist) From d5b25aa338c5895918489753256d015d9e93db15 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 09:34:15 +0100 Subject: [PATCH 104/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1a4cb36..a21d475 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -91,9 +91,6 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { git reset --hard git clean -fdx git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null """ ) } @@ -380,9 +377,9 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - tickets = tickets.sort { it.key } - echo "tickets map sorted:\n${tickets}" - for (ticketnumber in tickets) { + ticketssorted = tickets.sort { it.key } + echo "tickets map sorted:\n${ticketssorted}" + for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } writeFile(file: "ticketlist", text: ticketlist) From 79abd7ba56717effb371d2ba7aec72c472a599cb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 11:56:56 +0100 Subject: [PATCH 105/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a21d475..6df7368 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,7 +377,7 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - ticketssorted = tickets.sort { it.key } + ticketssorted = tickets.toSorted { a, b -> a.key <=> b.key } echo "tickets map sorted:\n${ticketssorted}" for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" From ae6c895c8ad3c7332d7fc0c47d6bfdacbaf52376 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 12:48:02 +0100 Subject: [PATCH 106/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6df7368..cb23c87 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -314,11 +314,11 @@ def addedcomponent(latermetaitems, key) { } def boolean componentreport(meta, componentdifferencerecords) { - def ticketed = [:] - def merges = [:] - def reverts = [:] - def unticketed = [:] - def tickets = [:] + def Map ticketed = [:] + def Map merges = [:] + def Map reverts = [:] + def Map unticketed = [:] + def Map tickets = [:] for (dr in componentdifferencerecords) { echo "Processing:\n${dr.dump()}" @@ -377,7 +377,7 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - ticketssorted = tickets.toSorted { a, b -> a.key <=> b.key } + ticketssorted = tickets.sort() echo "tickets map sorted:\n${ticketssorted}" for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" From 7a94c993c6f090cbaab143b117a3b900b5174c46 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:01:10 +0100 Subject: [PATCH 107/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 40 ++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index cb23c87..043a0ca 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,6 +313,32 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } +def componentmergereport(String componentreportfolder, merges) { + echo "Size of merges map: ${merges.size()}" + dir(componentreportfolder) { + String mergelist = "Merge list:\n" + echo "merges map:\n${merges}" + mergessorted = merges.sort() + echo "merges map sorted:\n${mergessorted}" + for (amerge in mergessorted) { + mergelist += "${amerge.key}\n" + } + writeFile(file: "mergelist.txt", text: mergelist) + } +} + +def componentticketsreport(String componentreportfolder, tickets) { + echo "Size of tickets map: ${tickets.size()}" + dir(componentreportfolder) { + String ticketlist = "Ticket list:\n" + ticketssorted = tickets.sort() + for (ticketnumber in ticketssorted) { + ticketlist += "${ticketnumber.key} (${tickets.value})\n" + } + writeFile(file: "ticketlist.txt", text: ticketlist) + } +} + def boolean componentreport(meta, componentdifferencerecords) { def Map ticketed = [:] def Map merges = [:] @@ -373,17 +399,9 @@ def boolean componentreport(meta, componentdifferencerecords) { } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" - echo "Size of tickets map: ${tickets.size()}" - dir(componentreportfolder) { - String ticketlist = "Ticket list:\n" - echo "tickets map:\n${tickets}" - ticketssorted = tickets.sort() - echo "tickets map sorted:\n${ticketssorted}" - for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" - } - writeFile(file: "ticketlist", text: ticketlist) - } + componentmergereport(componentreportfolder, merges) + componentticketreport(componentreportfolder, tickets) + } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key //tickets.sort { a, b -> a.value <=> b.value } From e982b587219d6958f933c1542f307ad3310e2ecd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:04:38 +0100 Subject: [PATCH 108/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 043a0ca..3d08eb8 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -400,7 +400,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" componentmergereport(componentreportfolder, merges) - componentticketreport(componentreportfolder, tickets) + componentticketsreport(componentreportfolder, tickets) } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key From fdec3b5601069aae577252beb20da369cdc2ce89 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:16:56 +0100 Subject: [PATCH 109/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3d08eb8..a384a07 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,13 +313,10 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def componentmergereport(String componentreportfolder, merges) { - echo "Size of merges map: ${merges.size()}" +def mergereport(String componentreportfolder, merges) { dir(componentreportfolder) { String mergelist = "Merge list:\n" - echo "merges map:\n${merges}" mergessorted = merges.sort() - echo "merges map sorted:\n${mergessorted}" for (amerge in mergessorted) { mergelist += "${amerge.key}\n" } @@ -327,13 +324,26 @@ def componentmergereport(String componentreportfolder, merges) { } } -def componentticketsreport(String componentreportfolder, tickets) { - echo "Size of tickets map: ${tickets.size()}" +def revertsreport(String componentreportfolder, reverts) { + echo "Size of reverts map: ${reverts.size()}" + dir(componentreportfolder) { + String revertlist = "Revert list:\n" + echo "reverts map:\n${reverts}" + revertssorted = reverts.sort() + echo "reverts map sorted:\n${revertssorted}" + for (arevert in revertssorted) { + revertlist += "${arevert.key}:\n${arevert.value.join("\n")}\n" + } + writeFile(file: "revertlist.txt", text: revertlist) + } +} + +def ticketsreport(String componentreportfolder, tickets) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber.key} (${tickets.value})\n" + ticketlist += "${ticketnumber.key} (${ticketnumber.value})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } @@ -399,8 +409,9 @@ def boolean componentreport(meta, componentdifferencerecords) { } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" - componentmergereport(componentreportfolder, merges) - componentticketsreport(componentreportfolder, tickets) + mergereport(componentreportfolder, merges) + revertsreport(componentreportfolder, reverts) + ticketsreport(componentreportfolder, tickets) } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key From 248803207feae3e8daecf8838d3e1aa8f5691b0c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:23:08 +0100 Subject: [PATCH 110/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a384a07..4c1bffe 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -338,6 +338,20 @@ def revertsreport(String componentreportfolder, reverts) { } } +def ticketedreport(String componentreportfolder, ticketed) { + echo "Size of ticketed map: ${ticketed.size()}" + dir(componentreportfolder) { + String ticketedlist = "Ticketed list:\n" + echo "ticketed map:\n${ticketed}" + ticketedsorted = ticketed.sort() + echo "ticketed map sorted:\n${ticketedsorted}" + for (aticketed in ticketedsorted) { + ticketedlist += "${aticketed.key}:\n${aticketed.value.join("\n")}\n" + } + writeFile(file: "ticketedlist.txt", text: ticketedlist) + } +} + def ticketsreport(String componentreportfolder, tickets) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" @@ -411,6 +425,7 @@ def boolean componentreport(meta, componentdifferencerecords) { String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" mergereport(componentreportfolder, merges) revertsreport(componentreportfolder, reverts) + ticketedreport(String componentreportfolder, ticketed) ticketsreport(componentreportfolder, tickets) } From 807ba4492dcdd3e4bed8f1dd9dd2778849ffb47d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:32:51 +0100 Subject: [PATCH 111/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 4c1bffe..fef482b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -425,7 +425,7 @@ def boolean componentreport(meta, componentdifferencerecords) { String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" mergereport(componentreportfolder, merges) revertsreport(componentreportfolder, reverts) - ticketedreport(String componentreportfolder, ticketed) + ticketedreport(componentreportfolder, ticketed) ticketsreport(componentreportfolder, tickets) } From 437a15986dd6d3f1cc3c7a7cb6ceca595b144502 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:52:26 +0100 Subject: [PATCH 112/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 55 ++++++++++++++++++------------ 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index fef482b..29c8a43 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,7 +313,7 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def mergereport(String componentreportfolder, merges) { +def mergereport(String componentreportfolder, merges, boolean iscomponent = true) { dir(componentreportfolder) { String mergelist = "Merge list:\n" mergessorted = merges.sort() @@ -324,13 +324,10 @@ def mergereport(String componentreportfolder, merges) { } } -def revertsreport(String componentreportfolder, reverts) { - echo "Size of reverts map: ${reverts.size()}" +def revertsreport(String componentreportfolder, reverts, boolean iscomponent = true) { dir(componentreportfolder) { String revertlist = "Revert list:\n" - echo "reverts map:\n${reverts}" revertssorted = reverts.sort() - echo "reverts map sorted:\n${revertssorted}" for (arevert in revertssorted) { revertlist += "${arevert.key}:\n${arevert.value.join("\n")}\n" } @@ -338,13 +335,10 @@ def revertsreport(String componentreportfolder, reverts) { } } -def ticketedreport(String componentreportfolder, ticketed) { - echo "Size of ticketed map: ${ticketed.size()}" +def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = true) { dir(componentreportfolder) { String ticketedlist = "Ticketed list:\n" - echo "ticketed map:\n${ticketed}" ticketedsorted = ticketed.sort() - echo "ticketed map sorted:\n${ticketedsorted}" for (aticketed in ticketedsorted) { ticketedlist += "${aticketed.key}:\n${aticketed.value.join("\n")}\n" } @@ -352,7 +346,21 @@ def ticketedreport(String componentreportfolder, ticketed) { } } -def ticketsreport(String componentreportfolder, tickets) { +def unticketedreport(String componentreportfolder, unticketed, boolean iscomponent = true) { + echo "Size of ticketed map: ${unticketed.size()}" + dir(componentreportfolder) { + String unticketedlist = "Unticketed list:\n" + echo "unticketed map:\n${unticketed}" + unticketedsorted = unticketed.sort() + echo "unticketed map sorted:\n${unticketedsorted}" + for (anunticketed in unticketedsorted) { + unticketedlist += "${anunticketed.key}\n" + } + writeFile(file: "unticketedlist.txt", text: unticketedlist) + } +} + +def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = true) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.sort() @@ -363,14 +371,14 @@ def ticketsreport(String componentreportfolder, tickets) { } } -def boolean componentreport(meta, componentdifferencerecords) { +def boolean report(meta, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] def Map merges = [:] def Map reverts = [:] def Map unticketed = [:] def Map tickets = [:] - for (dr in componentdifferencerecords) { + for (dr in differencerecords) { echo "Processing:\n${dr.dump()}" if (dr.type == "Merge") { @@ -422,16 +430,18 @@ def boolean componentreport(meta, componentdifferencerecords) { unticketed[dr.comment] = dr } } - String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" - mergereport(componentreportfolder, merges) - revertsreport(componentreportfolder, reverts) - ticketedreport(componentreportfolder, ticketed) - ticketsreport(componentreportfolder, tickets) - + String reportsubfolder = "" + if (iscomponent) { + reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" + } else { + reportsubfolder = "${reportfolder}/Combined" + } + mergereport(reportsubfolder, merges, iscomponent) + revertsreport(reportsubfolder, reverts, iscomponent) + ticketedreport(reportsubfolder, ticketed, iscomponent) + unticketedreport(reportsubfolder, unticketed, iscomponent) + ticketsreport(reportsubfolder, tickets, iscomponent) } -//m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key -//tickets.sort { a, b -> a.value <=> b.value } -//myMap.sort()*.key def compare(earlymetaitems, latermetaitems, key) { boolean result = true; @@ -447,7 +457,7 @@ def compare(earlymetaitems, latermetaitems, key) { (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) summaryappend(" Items: ${differencerecords.size()} ") - componentreport(earlymeta, differencerecords) + report(earlymeta, differencerecords) summaryappend("\n") return [result, differencerecords]; @@ -491,6 +501,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin summaryappend("\n") } } + report(meta, differencerecords, false) return [summarystring, markunstable, unstablereason] } From 5122aca1bf70a5b54137f21b000a2f3300bb5918 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 14:34:32 +0100 Subject: [PATCH 113/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 29c8a43..dc174a7 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -329,7 +329,10 @@ def revertsreport(String componentreportfolder, reverts, boolean iscomponent = t String revertlist = "Revert list:\n" revertssorted = reverts.sort() for (arevert in revertssorted) { - revertlist += "${arevert.key}:\n${arevert.value.join("\n")}\n" + revertlist += "${arevert.key}:\n" + for (revertlistcomment in arevert.value) { + revertlist += "${revertlistcomment.key} (${revertlistcomment.value})\n" + } } writeFile(file: "revertlist.txt", text: revertlist) } @@ -340,7 +343,10 @@ def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { - ticketedlist += "${aticketed.key}:\n${aticketed.value.join("\n")}\n" + ticketedlist += "${aticketed.key}:\n" + for (ticketedlistcomment in aticketed.value) { + ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" + } } writeFile(file: "ticketedlist.txt", text: ticketedlist) } @@ -355,6 +361,9 @@ def unticketedreport(String componentreportfolder, unticketed, boolean iscompone echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { unticketedlist += "${anunticketed.key}\n" + for (unticketedlistcomment in anunticketed.value) { + unticketedlist += "${unticketedlistcomment.key} (${unticketedlistcomment.value})\n" + } } writeFile(file: "unticketedlist.txt", text: unticketedlist) } @@ -390,10 +399,11 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (ticket in dr.jiraticketinfo) { if (!reverts.containsKey(ticket)) { echo "Creating reverts record for ${ticket}" - reverts[ticket] = [dr.comment] + reverts[ticket] = [:] + reverts[ticket][dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - reverts[ticket].add(dr.comment) + reverts[ticket][dr.comment] += 1 } if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" @@ -410,10 +420,11 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (ticket in dr.jiraticketinfo) { if (!ticketed.containsKey(ticket)) { echo "Creating ticketed record for ${ticket}" - ticketed[ticket] = [dr.comment] + ticketed[ticket] = [:] + ticketed[ticket][dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - ticketed[ticket].add(dr.comment) + ticketed[ticket][dr.comment] += 1 } if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" From 83e50fc58518da467ed89cc663fa85029854cc2c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:01:28 +0100 Subject: [PATCH 114/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dc174a7..c337e9a 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -397,14 +397,18 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (dr.type == "Revert") { echo "Adding to revert" for (ticket in dr.jiraticketinfo) { + def comments = [:] if (!reverts.containsKey(ticket)) { echo "Creating reverts record for ${ticket}" reverts[ticket] = [:] - reverts[ticket][dr.comment] = 1 + comments = reverts[ticket] + comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - reverts[ticket][dr.comment] += 1 + comments = reverts[ticket] + comments[dr.comment] += 1 } + reverts[ticket] = comments if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 @@ -418,14 +422,18 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (dr.type == "Ticketed") { echo "Adding to ticketed" for (ticket in dr.jiraticketinfo) { + def comments = [:] if (!ticketed.containsKey(ticket)) { echo "Creating ticketed record for ${ticket}" ticketed[ticket] = [:] - ticketed[ticket][dr.comment] = 1 + comments = ticketed[ticket] + comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - ticketed[ticket][dr.comment] += 1 + def comments = ticketed[ticket] + comments[dr.comment] += 1 } + ticketed[ticket] = comments if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 From 11fcc3279bf20505d00a7412813f9ec0cd8b51f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:03:20 +0100 Subject: [PATCH 115/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index c337e9a..c1449f5 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -430,7 +430,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - def comments = ticketed[ticket] + comments = ticketed[ticket] comments[dr.comment] += 1 } ticketed[ticket] = comments From 2f1a51c7582e9ebd417831abc92a1aeef9f09dee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:34:57 +0100 Subject: [PATCH 116/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index c1449f5..dfb75c4 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -405,8 +405,12 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - comments = reverts[ticket] - comments[dr.comment] += 1 + comments = ticketed[ticket] + if (!comments.containsKey(dr.comment)) { + comments[dr.comment] = 1 + } else { + comments[dr.comment] += 1 + } } reverts[ticket] = comments if (!tickets.containsKey(ticket)) { @@ -431,7 +435,11 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } else { echo "Adding to ticketed record for ${ticket}" comments = ticketed[ticket] - comments[dr.comment] += 1 + if (!comments.containsKey(dr.comment)) { + comments[dr.comment] = 1 + } else { + comments[dr.comment] += 1 + } } ticketed[ticket] = comments if (!tickets.containsKey(ticket)) { From b3b47d259328d157433d8b7cd3d6453322d4a447 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:45:00 +0100 Subject: [PATCH 117/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dfb75c4..cd988ac 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -380,6 +380,16 @@ def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = t } } +def differencerecordsreport(String reportsubfolder, differencerecords, , boolean iscomponent = true) { + dir(componentreportfolder) { + String recordlist = "" + for (record in differencerecords) { + recordlist += dr.dump() + } + writeFile(file: "recordlist.txt", text: recordlist) + } +} + def boolean report(meta, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] def Map merges = [:] @@ -388,8 +398,6 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { def Map tickets = [:] for (dr in differencerecords) { - echo "Processing:\n${dr.dump()}" - if (dr.type == "Merge") { echo "Adding to mergeinfo" merges[dr.mergeinfo] = dr @@ -468,6 +476,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { ticketedreport(reportsubfolder, ticketed, iscomponent) unticketedreport(reportsubfolder, unticketed, iscomponent) ticketsreport(reportsubfolder, tickets, iscomponent) + differencerecordsreport(reportsubfolder, differencerecords, iscomponent) } def compare(earlymetaitems, latermetaitems, key) { From 095ae84b697ad192ccd24ba54dbea017334cdc6f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:48:19 +0100 Subject: [PATCH 118/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index cd988ac..203d09b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -380,7 +380,7 @@ def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = t } } -def differencerecordsreport(String reportsubfolder, differencerecords, , boolean iscomponent = true) { +def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { dir(componentreportfolder) { String recordlist = "" for (record in differencerecords) { From e441a724ce95b0be3cc8c866258984992ea55ee2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 16:01:18 +0100 Subject: [PATCH 119/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 203d09b..e1d684b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,8 +313,8 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def mergereport(String componentreportfolder, merges, boolean iscomponent = true) { - dir(componentreportfolder) { +def mergereport(String reportsubfolder, merges, boolean iscomponent = true) { + dir(reportsubfolder) { String mergelist = "Merge list:\n" mergessorted = merges.sort() for (amerge in mergessorted) { @@ -324,8 +324,8 @@ def mergereport(String componentreportfolder, merges, boolean iscomponent = true } } -def revertsreport(String componentreportfolder, reverts, boolean iscomponent = true) { - dir(componentreportfolder) { +def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { + dir(reportsubfolder) { String revertlist = "Revert list:\n" revertssorted = reverts.sort() for (arevert in revertssorted) { @@ -338,8 +338,8 @@ def revertsreport(String componentreportfolder, reverts, boolean iscomponent = t } } -def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = true) { - dir(componentreportfolder) { +def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { + dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { @@ -352,9 +352,9 @@ def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = } } -def unticketedreport(String componentreportfolder, unticketed, boolean iscomponent = true) { +def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { echo "Size of ticketed map: ${unticketed.size()}" - dir(componentreportfolder) { + dir(reportsubfolder) { String unticketedlist = "Unticketed list:\n" echo "unticketed map:\n${unticketed}" unticketedsorted = unticketed.sort() @@ -369,8 +369,8 @@ def unticketedreport(String componentreportfolder, unticketed, boolean iscompone } } -def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = true) { - dir(componentreportfolder) { +def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { + dir(reportsubfolder) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { @@ -381,7 +381,7 @@ def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = t } def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { - dir(componentreportfolder) { + dir(reportsubfolder) { String recordlist = "" for (record in differencerecords) { recordlist += dr.dump() From e95197837f6a5660cbc7be2f6d6a3eef8b76f3ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 16:05:14 +0100 Subject: [PATCH 120/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e1d684b..34b90ec 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -383,7 +383,7 @@ def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { dir(reportsubfolder) { String recordlist = "" - for (record in differencerecords) { + for (dr in differencerecords) { recordlist += dr.dump() } writeFile(file: "recordlist.txt", text: recordlist) @@ -399,20 +399,16 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (dr in differencerecords) { if (dr.type == "Merge") { - echo "Adding to mergeinfo" merges[dr.mergeinfo] = dr } if (dr.type == "Revert") { - echo "Adding to revert" for (ticket in dr.jiraticketinfo) { def comments = [:] if (!reverts.containsKey(ticket)) { - echo "Creating reverts record for ${ticket}" reverts[ticket] = [:] comments = reverts[ticket] comments[dr.comment] = 1 } else { - echo "Adding to ticketed record for ${ticket}" comments = ticketed[ticket] if (!comments.containsKey(dr.comment)) { comments[dr.comment] = 1 @@ -422,26 +418,20 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } reverts[ticket] = comments if (!tickets.containsKey(ticket)) { - echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { - echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } - echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Ticketed") { - echo "Adding to ticketed" for (ticket in dr.jiraticketinfo) { def comments = [:] if (!ticketed.containsKey(ticket)) { - echo "Creating ticketed record for ${ticket}" ticketed[ticket] = [:] comments = ticketed[ticket] comments[dr.comment] = 1 } else { - echo "Adding to ticketed record for ${ticket}" comments = ticketed[ticket] if (!comments.containsKey(dr.comment)) { comments[dr.comment] = 1 @@ -451,17 +441,13 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } ticketed[ticket] = comments if (!tickets.containsKey(ticket)) { - echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { - echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } - echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Unticketed") { - echo "Adding to unticketed" unticketed[dr.comment] = dr } } From 15cf1e840a122536023ee50a9feebf45336309b9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:11:52 +0100 Subject: [PATCH 121/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- differences/metahierarchy.groovy | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 34b90ec..26230ab 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -384,7 +384,7 @@ def differencerecordsreport(String reportsubfolder, differencerecords, boolean i dir(reportsubfolder) { String recordlist = "" for (dr in differencerecords) { - recordlist += dr.dump() + recordlist += dr.dump() + "\n" } writeFile(file: "recordlist.txt", text: recordlist) } diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 8a42f5e..3392fdc 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -14,10 +14,15 @@ pipeline { script { SUMMARYDIR = "${WORKSPACE}/summary" REPORTDIR = "${WORKSPACE}/reports" + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + cleanfolders = [ "${WORKSPACE}/build_artifacts", - "${WORKSPACE}/early", - "${WORKSPACE}/later", + "${EARLYDIR}", + "${LATERDIR}", "${SUMMARYDIR}", "${REPORTDIR}", ] @@ -26,22 +31,14 @@ pipeline { } } } - stage('Checkout top level meta') { + stage('Obtain top level meta') { steps { script { - configurationhelper = autobuilder.configurationhelper() + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - - EARLYREVISION = RELEASE_EARLY_REVISION.trim() - EARLYROOTDIR = 'early' - EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" - configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) - - LATERREVISION = RELEASE_LATER_REVISION.trim() - LATERROOTDIR = 'later' - LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" - configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) + importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR) + importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR) } } } From 5520e3d09708a6c3b60223e4184aa4f296308a6e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:33:32 +0100 Subject: [PATCH 122/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 26230ab..9db7309 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -123,7 +123,7 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late } def String differencetext dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() } return differencetext } From 67a3458e96ed9584ff71c6b14a6360c148fdde9a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:40:04 +0100 Subject: [PATCH 123/177] Add more difference infrastructure --- differences/metahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 3392fdc..4bcdcdc 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -37,8 +37,8 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR) - importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR) + importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') + importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') } } } From 529067c8442a0610f765f3ac3302c8f04eedfb2f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:47:11 +0100 Subject: [PATCH 124/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9db7309..46a2943 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -275,6 +275,7 @@ def List processtext(meta, String text) { text.split('\n').each { DifferenceRecord dr = fillrecord(meta, it.trim()) differencerecords.add(dr) + echo "${dr.dump()}" } return differencerecords } From 1743fdef3dae5ff73771dea12b9ce7e7664f7e68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:50:16 +0100 Subject: [PATCH 125/177] Add more difference infrastructure --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 7e065d5..3ea97ec 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -251,7 +251,7 @@ def List convertSparseItemsString(String itemstopublish) { boolean nonempty = itemstopublish?.trim() if (nonempty) { - def String[] items = itemstopublish..trim().split(',') + def String[] items = itemstopublish.trim().split(',') def String allpaths = "" for(String item : items) { sparsepaths.add(item) From a95eb0c1a251d9d595438d17c593c6fe6e6fa795 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 18:06:21 +0100 Subject: [PATCH 126/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 46a2943..67b08f3 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -172,8 +172,7 @@ def extractmerges(String line) { def extractjiraticketinfo(String line) { def List jiratickets = [] - boolean amatch = false - while (!amatch) { + while (true) { line = line.trim() def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) @@ -181,7 +180,6 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) jiratickets.add(jiraticket) line = match1.group(2) - amatch = true } else { def pattern2 = '^(\\w+\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) @@ -189,7 +187,6 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) jiratickets.add(jiraticket) line = match2.group(2) - amatch = true } else { def pattern3 = '^(\\w+ \\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) @@ -197,8 +194,9 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) jiratickets.add(jiraticket) line = match3.group(2) - amatch = true } + } else { + break } } } From 81717b573f8e8e6e1b01de92d1b74d87e0f414a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 18:14:51 +0100 Subject: [PATCH 127/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 67b08f3..f98ee06 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -194,10 +194,10 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) jiratickets.add(jiraticket) line = match3.group(2) + } else { + break } - } else { - break - } + } } } boolean result = jiratickets.size() > 0 From e7bf289130db82cca79ac19992189e14519604b4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 18:17:54 +0100 Subject: [PATCH 128/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f98ee06..1b6055e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -259,7 +259,7 @@ def DifferenceRecord fillrecord(meta, line) { dr.comment = comment } else { dr.type = "Unticketed" - df.comment = line2 + dr.comment = line2 } } } From 29b0981c8a7a07e81e646193a633dce9d14c4965 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 10:55:38 +0100 Subject: [PATCH 129/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1b6055e..0a71913 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -359,10 +359,7 @@ def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = t unticketedsorted = unticketed.sort() echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key}\n" - for (unticketedlistcomment in anunticketed.value) { - unticketedlist += "${unticketedlistcomment.key} (${unticketedlistcomment.value})\n" - } + unticketedlist += "${anunticketed.key}\n${anunticketed.value()}" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } @@ -447,7 +444,12 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } } if (dr.type == "Unticketed") { - unticketed[dr.comment] = dr + if (!unticketed.containsKey(dr.comment)) { + unticketed[dr.comment] = 1 + } else { + unticketed[dr.comment] += 1 + } + } } } String reportsubfolder = "" From e8122dbab1f3eff143d55504e5d4762dc24b02de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 10:58:56 +0100 Subject: [PATCH 130/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 0a71913..ca2a633 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -446,9 +446,8 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (dr.type == "Unticketed") { if (!unticketed.containsKey(dr.comment)) { unticketed[dr.comment] = 1 - } else { - unticketed[dr.comment] += 1 - } + } else { + unticketed[dr.comment] += 1 } } } From e98a769dd1b0c29812e53c7c19e9988d602df2c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 11:02:20 +0100 Subject: [PATCH 131/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index ca2a633..379c482 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -359,7 +359,7 @@ def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = t unticketedsorted = unticketed.sort() echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key}\n${anunticketed.value()}" + unticketedlist += "${anunticketed.key}\n${anunticketed.value}" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } From 0280c4ec003d8c004382fb52370beaf6b5cef5e1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 11:14:16 +0100 Subject: [PATCH 132/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 379c482..f894e65 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -405,7 +405,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments = reverts[ticket] comments[dr.comment] = 1 } else { - comments = ticketed[ticket] + comments = reverts[ticket] if (!comments.containsKey(dr.comment)) { comments[dr.comment] = 1 } else { From 750955d8cdffc153bde3a6a4871ed3ac858b34a7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 12:26:55 +0100 Subject: [PATCH 133/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 57 ++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f894e65..c29a278 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -169,6 +169,21 @@ def extractmerges(String line) { return [false, line] } +def jiraticketexists(String jiraticket) { + def String URI = "${JIRA_URI}/browse/${jiraticket}" + def get = new URL(URI).openConnection(); + get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) + get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) + get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) + get.setRequestProperty( 'Accept', 'application/json' ) + def getRC = get.getResponseCode(); + + if(!getRC.equals(200)) { + return [false, ''] + } + return [true, URI] +} + def extractjiraticketinfo(String line) { def List jiratickets = [] @@ -178,22 +193,37 @@ def extractjiraticketinfo(String line) { def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) - jiratickets.add(jiraticket) - line = match1.group(2) - } else { - def pattern2 = '^(\\w+\\d+)(.*)' //GIS1234 smb1234 + (exists, URI) = jiraticketexists(jiraticket) + if (exists) { + jiratickets.add([jiraticket, URI]) + line = match1.group(2) + } else { + break + } + } else { + def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) if (match2.find()) { - jiraticket = match2.group(1) - jiratickets.add(jiraticket) - line = match2.group(2) + jiraticket = match2.group(1) + "-" + match2.group(2) + (exists, URI) = jiraticketexists(jiraticket) + if (exists) { + jiratickets.add([jiraticket, URI]) + line = match2.group(3) + } else { + break + } } else { - def pattern3 = '^(\\w+ \\d+)(.*)' //GIS 1234 smb 1234 + def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) if (match3.find()) { - jiraticket = match3.group(1) - jiratickets.add(jiraticket) - line = match3.group(2) + jiraticket = match3.group(1) + "-" + match3.group(2) + (exists, URI) = jiraticketexists(jiraticket) + if (exists) { + jiratickets.add([jiraticket, URI]) + line = match3.group(3) + } else { + break + } } else { break } @@ -352,14 +382,11 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) } def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { - echo "Size of ticketed map: ${unticketed.size()}" dir(reportsubfolder) { String unticketedlist = "Unticketed list:\n" - echo "unticketed map:\n${unticketed}" unticketedsorted = unticketed.sort() - echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key}\n${anunticketed.value}" + unticketedlist += "${anunticketed.key} (${anunticketed.value})\n" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } From d755e394e97e5eef8ac46e191a1c3b01f0a81e48 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 12:34:33 +0100 Subject: [PATCH 134/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index c29a278..449a162 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -186,7 +186,9 @@ def jiraticketexists(String jiraticket) { def extractjiraticketinfo(String line) { def List jiratickets = [] - + boolean exists = false + String URI = '' + while (true) { line = line.trim() def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 From 2cf59337afb680a4551bc7fe787dff099d03bcd7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 13:07:57 +0100 Subject: [PATCH 135/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 449a162..43e42cb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -197,7 +197,8 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - jiratickets.add([jiraticket, URI]) + def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] + jiratickets.add(jiraticketmap) line = match1.group(2) } else { break @@ -209,7 +210,8 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - jiratickets.add([jiraticket, URI]) + def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] + jiratickets.add(jiraticketmap) line = match2.group(3) } else { break @@ -221,7 +223,8 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - jiratickets.add([jiraticket, URI]) + def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] + jiratickets.add(jiraticketmap) line = match3.group(3) } else { break From 86d40f76887a5af654c7ac91e789fdf3222f5cc0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 14:46:50 +0100 Subject: [PATCH 136/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 43e42cb..7d66151 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -361,7 +361,7 @@ def mergereport(String reportsubfolder, merges, boolean iscomponent = true) { def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { dir(reportsubfolder) { String revertlist = "Revert list:\n" - revertssorted = reverts.sort() + revertssorted = reverts.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (arevert in revertssorted) { revertlist += "${arevert.key}:\n" for (revertlistcomment in arevert.value) { @@ -375,7 +375,7 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" - ticketedsorted = ticketed.sort() + ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (aticketed in ticketedsorted) { ticketedlist += "${aticketed.key}:\n" for (ticketedlistcomment in aticketed.value) { @@ -400,7 +400,7 @@ def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = t def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { dir(reportsubfolder) { String ticketlist = "Ticket list:\n" - ticketssorted = tickets.sort() + ticketssorted = tickets.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber.key} (${ticketnumber.value})\n" } From 19cfa866a4a422a67676bb21c187e08bd882e97d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:05:27 +0100 Subject: [PATCH 137/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 7d66151..d959b0f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -444,7 +444,8 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments[dr.comment] += 1 } } - reverts[ticket] = comments + reverts[ticket] = comments + echo "ticket: ${ticket} revert[ticket]: ${reverts[ticket]}" if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 } else { From 783f5e2b42c6df2113565a445f3da1a1430495b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:15:18 +0100 Subject: [PATCH 138/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index d959b0f..307fd48 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -363,7 +363,7 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { String revertlist = "Revert list:\n" revertssorted = reverts.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (arevert in revertssorted) { - revertlist += "${arevert.key}:\n" + revertlist += "${arevert.key['jiraticket']}: ${arevert.key['uri']}\n" for (revertlistcomment in arevert.value) { revertlist += "${revertlistcomment.key} (${revertlistcomment.value})\n" } @@ -377,7 +377,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (aticketed in ticketedsorted) { - ticketedlist += "${aticketed.key}:\n" + ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" for (ticketedlistcomment in aticketed.value) { ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" } @@ -386,6 +386,8 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) } } +ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] + def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { dir(reportsubfolder) { String unticketedlist = "Unticketed list:\n" @@ -402,7 +404,7 @@ def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber.key} (${ticketnumber.value})\n" + ticketlist += "${ticketnumber.key['jiraticket']}: ${ticketnumber.key['uri']} (${ticketnumber.value})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } From 1ff0e8298b30e5b42b5c3869c6af0d80e3ab52fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:17:10 +0100 Subject: [PATCH 139/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 307fd48..bb86e36 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -386,7 +386,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) } } -ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] +//ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { dir(reportsubfolder) { From 05271844bbd4f43c2863faebe16d390aee8c4f77 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:23:30 +0100 Subject: [PATCH 140/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index bb86e36..e35cd8d 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,6 +377,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (aticketed in ticketedsorted) { + echo "aticketed: ${aticketed}" ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" for (ticketedlistcomment in aticketed.value) { ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" @@ -447,7 +448,6 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } } reverts[ticket] = comments - echo "ticket: ${ticket} revert[ticket]: ${reverts[ticket]}" if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 } else { From d154a643e2bfe378b9472e865b8cd657c1f8e56a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:31:40 +0100 Subject: [PATCH 141/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e35cd8d..a7ca611 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -376,6 +376,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + echo "ticketedsorted: ${ticketedsorted}" for (aticketed in ticketedsorted) { echo "aticketed: ${aticketed}" ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" From 79a9eecc51aa05306b088ae1c8d21b2eb94ba033 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:37:27 +0100 Subject: [PATCH 142/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a7ca611..adefdfc 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -375,7 +375,8 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" - ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + echo "ticketed: ${ticketed}" + ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } echo "ticketedsorted: ${ticketedsorted}" for (aticketed in ticketedsorted) { echo "aticketed: ${aticketed}" From 15ae9f815f3680b0a5103929eace3aca17eec691 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 16:25:52 +0100 Subject: [PATCH 143/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index adefdfc..1b297de 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -375,8 +375,14 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" - echo "ticketed: ${ticketed}" - ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + echo "ticketed: ${ticketed}" + for (ticket in ticketed) { + echo "|${ticket.key['jiraticket']|\n|${ticket.key['uri']|\n" + for (comment in ticket.value) { + echo "|${comment.key}|${comment.value}|\n" + } + } + ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } echo "ticketedsorted: ${ticketedsorted}" for (aticketed in ticketedsorted) { echo "aticketed: ${aticketed}" From 39b849c61caf2315b059c04fd3f6557fe9359975 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 16:38:32 +0100 Subject: [PATCH 144/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1b297de..5d290b9 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,7 +377,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" echo "ticketed: ${ticketed}" for (ticket in ticketed) { - echo "|${ticket.key['jiraticket']|\n|${ticket.key['uri']|\n" + echo "|${ticket.key['jiraticket']}|\n|${ticket.key['uri']}|\n" for (comment in ticket.value) { echo "|${comment.key}|${comment.value}|\n" } From 9062919a7f4b1a43d6cd87d65218574eaeb1c3f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 16:56:19 +0100 Subject: [PATCH 145/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5d290b9..06ade92 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,9 +377,9 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" echo "ticketed: ${ticketed}" for (ticket in ticketed) { - echo "|${ticket.key['jiraticket']}|\n|${ticket.key['uri']}|\n" + echo "|${ticket.key['jiraticket']}(${ticket.key['jiraticket'].getClass()})|\n|${ticket.key['uri']}(${ticket.key['uri'].getClass()})|\n" for (comment in ticket.value) { - echo "|${comment.key}|${comment.value}|\n" + echo "|${comment.key}(${comment.key.getClass()})|${comment.value}(${comment.value.getClass()})|\n" } } ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } From 1b247ccb774aed23a3dde6a33050c4c800058459 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:38:16 +0100 Subject: [PATCH 146/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 262 +++++++++++++++++++++-------- 1 file changed, 195 insertions(+), 67 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 06ade92..9adc784 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -184,6 +184,16 @@ def jiraticketexists(String jiraticket) { return [true, URI] } +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } +} + def extractjiraticketinfo(String line) { def List jiratickets = [] boolean exists = false @@ -197,8 +207,10 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] - jiratickets.add(jiraticketmap) + def JiraTicketInfo j = new JiraTicketInfo + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) line = match1.group(2) } else { break @@ -210,8 +222,10 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] - jiratickets.add(jiraticketmap) + def JiraTicketInfo j = new JiraTicketInfo + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) line = match2.group(3) } else { break @@ -223,8 +237,10 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] - jiratickets.add(jiraticketmap) + def JiraTicketInfo j = new JiraTicketInfo + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) line = match3.group(3) } else { break @@ -347,73 +363,137 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def mergereport(String reportsubfolder, merges, boolean iscomponent = true) { +class MergeReportRecord { + String comment + int duplicates + + MergeReportRecord() { + comment = '' + duplicates = 0 + } +} + +def mergereport(String reportsubfolder, LinkedHashMap merges, boolean iscomponent = true) { dir(reportsubfolder) { - String mergelist = "Merge list:\n" - mergessorted = merges.sort() + def String mergelist = "Merge list:\n" + def mergessorted = merges.sort() for (amerge in mergessorted) { - mergelist += "${amerge.key}\n" + def MergeReportRecord m = amerge.value + mergelist += "${m.comment} (${m.duplicates()})\n" } writeFile(file: "mergelist.txt", text: mergelist) } } -def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { +class CommentsRecord { + String comment + int duplicates + + CommentsRecord() { + comment = '' + duplicates = 0 + } +} + +class RevertsReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + RevertsReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def revertsreport(String reportsubfolder, LinkedHashMap reverts, boolean iscomponent = true) { dir(reportsubfolder) { - String revertlist = "Revert list:\n" - revertssorted = reverts.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + def String revertlist = "Revert list:\n" + def revertssorted = reverts.sort() for (arevert in revertssorted) { - revertlist += "${arevert.key['jiraticket']}: ${arevert.key['uri']}\n" - for (revertlistcomment in arevert.value) { - revertlist += "${revertlistcomment.key} (${revertlistcomment.value})\n" + def RevertsReportRecord r = arevert.value + revertlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + revertlist += "${commentrecord.comment} (${commentrecord.duplicates})\n" } } writeFile(file: "revertlist.txt", text: revertlist) } } -def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { +class TicketedReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + TicketedReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def ticketedreport(String reportsubfolder, LinkedHashMap ticketed, boolean iscomponent = true) { dir(reportsubfolder) { - String ticketedlist = "Ticketed list:\n" - echo "ticketed: ${ticketed}" - for (ticket in ticketed) { - echo "|${ticket.key['jiraticket']}(${ticket.key['jiraticket'].getClass()})|\n|${ticket.key['uri']}(${ticket.key['uri'].getClass()})|\n" - for (comment in ticket.value) { - echo "|${comment.key}(${comment.key.getClass()})|${comment.value}(${comment.value.getClass()})|\n" - } - } - ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } - echo "ticketedsorted: ${ticketedsorted}" + def String ticketedlist = "Ticketed list:\n" + def ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { - echo "aticketed: ${aticketed}" - ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" - for (ticketedlistcomment in aticketed.value) { - ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" + def TicketedReportRecord r = aticket.value + ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + ticketedlist += "${commentrecord.comment} (${commentrecord.duplicates})\n" } } writeFile(file: "ticketedlist.txt", text: ticketedlist) } } -//ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] +class UnticketedReportRecord { + String comment + int duplicates + + UnticketedReportRecord() { + comment = '' + duplicates = 0 + } +} -def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { +def unticketedreport(String reportsubfolder, LinkedHashMap unticketed, boolean iscomponent = true) { dir(reportsubfolder) { - String unticketedlist = "Unticketed list:\n" - unticketedsorted = unticketed.sort() + def String unticketedlist = "Unticketed list:\n" + def unticketedsorted = unticketed.sort() for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key} (${anunticketed.value})\n" + UnticketedReportRecord u = anunticketed.value + unticketedlist += "${u.comment} (${u.duplicates})\n" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } } +class TicketsReportRecord { + String jiraticket + String ticketURI + int duplicates + + TicketsReportRecord() { + jiraticket = '' + ticketURI = '' + duplicates = 0 + } +} + def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { dir(reportsubfolder) { String ticketlist = "Ticket list:\n" - ticketssorted = tickets.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber.key['jiraticket']}: ${ticketnumber.key['uri']} (${ticketnumber.value})\n" + TicketsReportRecord t = ticketnumber.value + ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } @@ -438,60 +518,108 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (dr in differencerecords) { if (dr.type == "Merge") { - merges[dr.mergeinfo] = dr + def MergeReportRecord m + if (!merges.containsKey(dr.mergeinfo)) { + m = new MergeReportRecord() + m.comment = dr.mergeinfo + m.duplicate = 1 + } else { + m = merges[dr.mergeinfo] + m.duplicate += 1 + } + merges[dr.mergeinfo] = m } if (dr.type == "Revert") { + def RevertsReportRecord r for (ticket in dr.jiraticketinfo) { - def comments = [:] - if (!reverts.containsKey(ticket)) { - reverts[ticket] = [:] - comments = reverts[ticket] - comments[dr.comment] = 1 + JiraTicketInfo j = ticket + if (!reverts.containsKey(j.jiranumber)) { + r = new RevertsReportRecord() + r.jiraticket = j.jiranumber + r.ticketURI = j.uri + CommentsRecord c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c } else { - comments = reverts[ticket] - if (!comments.containsKey(dr.comment)) { - comments[dr.comment] = 1 + r = reverts[j.jiranumber] + def CommentsRecord c + if (!r.commentscollection.containsKey(dr.comment)) { + c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c } else { - comments[dr.comment] += 1 + c = r.commentscollection[dr.comment] + c.duplicates += 1 } + r.commentscollection[dr.comment] = c } - reverts[ticket] = comments - if (!tickets.containsKey(ticket)) { - tickets[ticket] = 1 + reverts[j.jiranumber] = r + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 } else { - tickets[ticket] += 1 + t = tickets[j.jiranumber] + t.duplicates += 1 } + tickets[j.jiranumber] = t } } if (dr.type == "Ticketed") { + def TicketedReportRecord t for (ticket in dr.jiraticketinfo) { - def comments = [:] - if (!ticketed.containsKey(ticket)) { - ticketed[ticket] = [:] - comments = ticketed[ticket] - comments[dr.comment] = 1 + JiraTicketInfo j = ticket + if (!ticketed.containsKey(j.jiranumber)) { + t = new TicketedReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + CommentsRecord c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c } else { - comments = ticketed[ticket] - if (!comments.containsKey(dr.comment)) { - comments[dr.comment] = 1 + t = ticketed[j.jiranumber] + def CommentsRecord c + if (!t.commentscollection.containsKey(dr.comment)) { + c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c } else { - comments[dr.comment] += 1 + c = t.commentscollection[dr.comment] + c.duplicates += 1 } + t.commentscollection[dr.comment] = c } - ticketed[ticket] = comments - if (!tickets.containsKey(ticket)) { - tickets[ticket] = 1 + ticketed[j.jiranumber] = t + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 } else { - tickets[ticket] += 1 + t = tickets[j.jiranumber] + t.duplicates += 1 } + tickets[j.jiranumber] = t } } if (dr.type == "Unticketed") { + UnticketedReportRecord u if (!unticketed.containsKey(dr.comment)) { - unticketed[dr.comment] = 1 + u = new UnticketedReportRecord() + u.comment = dr.comment + u.duplicates = 1 } else { - unticketed[dr.comment] += 1 + u = unticketed[dr.comment] + u.duplicates += 1 } + unticketed[dr.comment] = u } } String reportsubfolder = "" From c7536f377b217c189eb09499d12838bc1cf44312 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:43:43 +0100 Subject: [PATCH 147/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9adc784..a542edb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -207,7 +207,7 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def JiraTicketInfo j = new JiraTicketInfo + def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) @@ -222,7 +222,7 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def JiraTicketInfo j = new JiraTicketInfo + def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) @@ -237,7 +237,7 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def JiraTicketInfo j = new JiraTicketInfo + def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) From b3188c775247a4a2de587a47b00571ac222cd716 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:47:56 +0100 Subject: [PATCH 148/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a542edb..3a2538e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -596,17 +596,17 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { t.commentscollection[dr.comment] = c } ticketed[j.jiranumber] = t - def TicketsReportRecord t + def TicketsReportRecord ti if (!tickets.containsKey(j.jiranumber)) { - t = new TicketsReportRecord() - t.jiraticket = j.jiranumber - t.ticketURI = j.uri - t.duplicates = 1 + ti = new TicketsReportRecord() + ti.jiraticket = j.jiranumber + ti.ticketURI = j.uri + ti.duplicates = 1 } else { - t = tickets[j.jiranumber] - t.duplicates += 1 + ti = tickets[j.jiranumber] + ti.duplicates += 1 } - tickets[j.jiranumber] = t + tickets[j.jiranumber] = ti } } if (dr.type == "Unticketed") { From 28dc2231bf161b289c5b678603bba78e6b91c1a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:56:07 +0100 Subject: [PATCH 149/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3a2538e..a1ed656 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -192,6 +192,10 @@ class JiraTicketInfo { jiranumber = '' uri = '' } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } } def extractjiraticketinfo(String line) { @@ -522,10 +526,10 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (!merges.containsKey(dr.mergeinfo)) { m = new MergeReportRecord() m.comment = dr.mergeinfo - m.duplicate = 1 + m.duplicates = 1 } else { m = merges[dr.mergeinfo] - m.duplicate += 1 + m.duplicates += 1 } merges[dr.mergeinfo] = m } From 7ff591f1fa1db05fa59294bdc0a8aba2ed382560 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:04:00 +0100 Subject: [PATCH 150/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a1ed656..4401059 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -10,6 +10,20 @@ tempfolder = "${WORKSPACE}/temp" reportfolder = "${WORKSPACE}/reports" +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } +} + class DifferenceRecord { String productname String subcomponentname @@ -19,7 +33,7 @@ class DifferenceRecord { String sha String type String mergeinfo - List jiraticketinfo + List jiraticketinfo String comment DifferenceRecord() { @@ -31,7 +45,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" - jiraticketinfo = [] + jiraticketinfo = new List comment = "" } @@ -184,20 +198,6 @@ def jiraticketexists(String jiraticket) { return [true, URI] } -class JiraTicketInfo { - String jiranumber - String uri - - JiraTicketInfo() { - jiranumber = '' - uri = '' - } - - def String dump() { - return "jiranumber: ${jiranumber} uri: ${uri}" - } -} - def extractjiraticketinfo(String line) { def List jiratickets = [] boolean exists = false @@ -541,7 +541,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { r = new RevertsReportRecord() r.jiraticket = j.jiranumber r.ticketURI = j.uri - CommentsRecord c = mew CommentsRecord() + CommentsRecord c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 r.commentscollection[dr.comment] = c @@ -549,7 +549,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { r = reverts[j.jiranumber] def CommentsRecord c if (!r.commentscollection.containsKey(dr.comment)) { - c = mew CommentsRecord() + c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 r.commentscollection[dr.comment] = c @@ -581,7 +581,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { t = new TicketedReportRecord() t.jiraticket = j.jiranumber t.ticketURI = j.uri - CommentsRecord c = mew CommentsRecord() + CommentsRecord c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 t.commentscollection[dr.comment] = c @@ -589,7 +589,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { t = ticketed[j.jiranumber] def CommentsRecord c if (!t.commentscollection.containsKey(dr.comment)) { - c = mew CommentsRecord() + c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 t.commentscollection[dr.comment] = c From 348620371002aa07e3f964979188d75bfb2a08cf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:15:41 +0100 Subject: [PATCH 151/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 4401059..3916d61 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -45,7 +45,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" - jiraticketinfo = new List + jiraticketinfo = new List() comment = "" } From 4b2b4018f787e05f177d68d559e2acf7e74ffda7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:29:06 +0100 Subject: [PATCH 152/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3916d61..1fb6444 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -45,7 +45,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" - jiraticketinfo = new List() + jiraticketinfo = [] comment = "" } From 5c1c461bcd07812720d7c5a25dfd904cf60378a3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:37:55 +0100 Subject: [PATCH 153/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1fb6444..08017cb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -50,7 +50,11 @@ class DifferenceRecord { } def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" + String jiraticketstring = '' + for (j in jiraticketinfo) { + jiraticketstring += j.dump() + " " + } + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -383,7 +387,7 @@ def mergereport(String reportsubfolder, LinkedHashMap def mergessorted = merges.sort() for (amerge in mergessorted) { def MergeReportRecord m = amerge.value - mergelist += "${m.comment} (${m.duplicates()})\n" + mergelist += "${m.comment} (${m.duplicates})\n" } writeFile(file: "mergelist.txt", text: mergelist) } From bac02149cecb8723e138ce062f6e51035dd2524d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:48:53 +0100 Subject: [PATCH 154/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 08017cb..20c2af6 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -425,7 +425,8 @@ def revertsreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 21:50:21 +0100 Subject: [PATCH 155/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 20c2af6..63ba65e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -425,7 +425,7 @@ def revertsreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:03:12 +0100 Subject: [PATCH 156/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 63ba65e..8bd1a74 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -209,6 +209,11 @@ def extractjiraticketinfo(String line) { while (true) { line = line.trim() + def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line + def commamatch = (line =~ commapattern) + if (commamatch.find()) { + line = commamatch.group(1).trim() + } def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) if (match1.find()) { @@ -450,7 +455,7 @@ def ticketedreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:11:00 +0100 Subject: [PATCH 157/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 8bd1a74..39b57e6 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -460,7 +460,8 @@ def ticketedreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:42:21 +0100 Subject: [PATCH 158/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 39b57e6..60a4c66 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -464,6 +464,7 @@ def ticketedreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:47:09 +0100 Subject: [PATCH 159/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 60a4c66..3c4aefb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -337,7 +337,7 @@ def List processtext(meta, String text) { text.split('\n').each { DifferenceRecord dr = fillrecord(meta, it.trim()) differencerecords.add(dr) - echo "${dr.dump()}" + //echo "${dr.dump()}" } return differencerecords } From 39e4ff5409bd48803dd52e493f7685505c8f630e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 22:50:03 +0100 Subject: [PATCH 160/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3c4aefb..9443fd1 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -433,6 +433,7 @@ def revertsreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 23:01:36 +0100 Subject: [PATCH 161/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9443fd1..b4f438f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -189,7 +189,8 @@ def extractmerges(String line) { def jiraticketexists(String jiraticket) { def String URI = "${JIRA_URI}/browse/${jiraticket}" - def get = new URL(URI).openConnection(); + def String URIforsummary = "${URI}?fields=summary" + def get = new URL(URIforsummary).openConnection(); get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) From b2a891e9cc36fff7b28f5ebec31269f13f85cbc8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 23:17:12 +0100 Subject: [PATCH 162/177] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index b4f438f..6d5c2fb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -521,7 +521,7 @@ def differencerecordsreport(String reportsubfolder, differencerecords, boolean i dir(reportsubfolder) { String recordlist = "" for (dr in differencerecords) { - recordlist += dr.dump() + "\n" + recordlist += dr.dump() + "\n\n" } writeFile(file: "recordlist.txt", text: recordlist) } From 4a796b33abb626ada8b8e45e47058a53eeada34f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 23:57:39 +0100 Subject: [PATCH 163/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6d5c2fb..0b056cf 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -101,6 +101,7 @@ def showmetaitems(metaitems) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" if (sourcerepo) { // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component if (fileExists(checkoutfolder)) { From e40949419a5d18721bc66f27b73b0342242d773d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 00:06:45 +0100 Subject: [PATCH 164/177] Add more difference infrastructure --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 0b056cf..3c97076 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -127,6 +127,7 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + echo "Cloning/updating source repo in folder: ${uniquefolderforsourcerepo}" checkoutsourcerepo(meta, uniquefolderforsourcerepo) return uniquefolderforsourcerepo From 85d7f751da8e638bd2d77331810978f3f29d4a0f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 00:27:10 +0100 Subject: [PATCH 165/177] Handle components that have no source repo --- differences/dometahierarchy.groovy | 57 ++++++++++++++++++------------ 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3c97076..6336c33 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -99,27 +99,32 @@ def showmetaitems(metaitems) { echo info } -def checkoutsourcerepo(buildmeta, String checkoutfolder) { +def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { + boolean checkedout = false def String sourcerepo = buildmeta.gitinfo.remoteurl - echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" - if (sourcerepo) { - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - """ - ) + if (sourcerepo?.trim()) { + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + """ + ) + } } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } } } + checkedout = true } + return checkedout } def String getcomponent(meta) { @@ -127,10 +132,9 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - echo "Cloning/updating source repo in folder: ${uniquefolderforsourcerepo}" - checkoutsourcerepo(meta, uniquefolderforsourcerepo) + bool checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) - return uniquefolderforsourcerepo + return [checkedout, uniquefolderforsourcerepo] } def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { @@ -665,12 +669,19 @@ def compare(earlymetaitems, latermetaitems, key) { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") - def uniquefolderforsourcerepo = getcomponent(earlymeta) + def boolean repoavailable + def String uniquefolderforsourcerepo + (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) - (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + List differencerecords = [] + if (repoavailable) { + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - summaryappend(" Items: ${differencerecords.size()} ") - report(earlymeta, differencerecords) + summaryappend(" Items: ${differencerecords.size()} ") + report(earlymeta, differencerecords) + } else { + summaryappend(" No source repo ") + } summaryappend("\n") return [result, differencerecords]; From a0fcc01a0d9d808e8453b523458ab89bc48f9d32 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 00:30:39 +0100 Subject: [PATCH 166/177] Handle components that have no source repo --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6336c33..b7d4615 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -132,7 +132,7 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - bool checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) + boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) return [checkedout, uniquefolderforsourcerepo] } From 06591a0daaddcc2da9570ce230c86216c355c1ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 07:44:38 +0100 Subject: [PATCH 167/177] Handle the no change case - the early and later sha codes match --- differences/dometahierarchy.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index b7d4615..e515c06 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -342,9 +342,12 @@ def List processtext(meta, String text) { def List differencerecords = [] text.split('\n').each { - DifferenceRecord dr = fillrecord(meta, it.trim()) - differencerecords.add(dr) - //echo "${dr.dump()}" + def line = it + if (line?.trim()) { + def DifferenceRecord dr = fillrecord(meta, line.trim()) + differencerecords.add(dr) + //echo "${dr.dump()}" + } } return differencerecords } @@ -363,6 +366,9 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) differencerecords = processtext(earlymeta, differencetext) + if (earlyrevision == laterrevision) { + summaryappend(" No change") + } } else { summaryappend("The early and later git urls don't match") echo "The early and later git urls don't match" From aecd252a57158eb20babe62cc40bb43be5e433b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 09:04:07 +0100 Subject: [PATCH 168/177] Normalise Jira ticket number --- differences/dometahierarchy.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e515c06..dccb5b4 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -225,6 +225,7 @@ def extractjiraticketinfo(String line) { def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) + jiraticket = jiraticket.toUpperCase() (exists, URI) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() @@ -240,6 +241,7 @@ def extractjiraticketinfo(String line) { def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) + "-" + match2.group(2) + jiraticket = jiraticket.toUpperCase() (exists, URI) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() @@ -255,6 +257,7 @@ def extractjiraticketinfo(String line) { def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) + "-" + match3.group(2) + jiraticket = jiraticket.toUpperCase() (exists, URI) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() From 877f022678fc67bc971cb69a18b88b8c2b508a79 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 10:44:26 +0100 Subject: [PATCH 169/177] Handle Jira ticket redirects - as a result of a jira ticket being moved to another project --- differences/dometahierarchy.groovy | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dccb5b4..9258da3 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,3 +1,5 @@ +import groovy.json.JsonSlurper + markunstable = false unstablereason = "" @@ -203,10 +205,15 @@ def jiraticketexists(String jiraticket) { get.setRequestProperty( 'Accept', 'application/json' ) def getRC = get.getResponseCode(); + def String redirectedjiraticket = jiraticket if(!getRC.equals(200)) { - return [false, ''] + return [false, '', jiraticket] + } else { + def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) + echo "Response: ${jsonresponse}" + redirectedjiraticket = jsonresponse['key'] } - return [true, URI] + return [true, URI, redirectedjiraticket] } def extractjiraticketinfo(String line) { @@ -225,8 +232,7 @@ def extractjiraticketinfo(String line) { def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) - jiraticket = jiraticket.toUpperCase() - (exists, URI) = jiraticketexists(jiraticket) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket @@ -241,8 +247,7 @@ def extractjiraticketinfo(String line) { def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) + "-" + match2.group(2) - jiraticket = jiraticket.toUpperCase() - (exists, URI) = jiraticketexists(jiraticket) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket @@ -257,8 +262,7 @@ def extractjiraticketinfo(String line) { def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) + "-" + match3.group(2) - jiraticket = jiraticket.toUpperCase() - (exists, URI) = jiraticketexists(jiraticket) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket From 8ce4442dd68c7e422a1f4c2abcb1657062293975 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 12:15:53 +0100 Subject: [PATCH 170/177] Handle Jira ticket redirects - as a result of a jira ticket being moved to another project --- differences/dometahierarchy.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9258da3..11b3733 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -197,8 +197,8 @@ def extractmerges(String line) { def jiraticketexists(String jiraticket) { def String URI = "${JIRA_URI}/browse/${jiraticket}" - def String URIforsummary = "${URI}?fields=summary" - def get = new URL(URIforsummary).openConnection(); + def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" + def get = new URL(RestfulURI).openConnection(); get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) @@ -210,7 +210,6 @@ def jiraticketexists(String jiraticket) { return [false, '', jiraticket] } else { def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) - echo "Response: ${jsonresponse}" redirectedjiraticket = jsonresponse['key'] } return [true, URI, redirectedjiraticket] From 8af57f80b93098579f7c2e4556c0c6eacd0da00c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 18 Jul 2019 11:22:06 +0100 Subject: [PATCH 171/177] Provide logging to debug why import of Print Server artifacts results in duff associated meta data --- meta/folder.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index f8c033b..b559458 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -571,6 +571,7 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJson(String filepath) { + script.echo("Deserialising JSON at file: ${filepath}" def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } @@ -617,6 +618,8 @@ class MetaInfo extends GISJsonSerialised { def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") } RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) From dbe2cdd5fdbef3e21bf5d7c0a97f69503576ea11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 18 Jul 2019 11:34:23 +0100 Subject: [PATCH 172/177] Provide logging to debug why import of Print Server artifacts results in duff associated meta data --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index b559458..ddc99c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -571,7 +571,7 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJson(String filepath) { - script.echo("Deserialising JSON at file: ${filepath}" + script.echo("Deserialising JSON at file: ${filepath}") def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } From aaba6bf78ef17b1c84d33490611bb17f4d9ba26a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:43:21 +0100 Subject: [PATCH 173/177] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 738c04a..1898594 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -92,9 +92,10 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { } } -def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { +def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") + bool exceptionduringcopyartifacts = false for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -102,22 +103,30 @@ def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, Strin } // Copy relevant artifacts to temp folder echo "Copying artifacts: ${filterpath} to ${tempfolder}" - copyArtifacts( - filter: filterpath, - projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - ], - target: tempfolder - ) - // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.offsetpath - dir(originalrootdir) { - sh("mv * ${repofolder}") + try { + copyArtifacts( + filter: filterpath, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + ], + target: tempfolder + ) + } catch (Exception e) { + echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" + exceptionduringcopyartifacts = true + summaryappend("Does not exist: ${artifactfilter}," + } else { + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } } summaryappend("${artifactfilter},") } + return exceptionduringcopyartifacts } def releasegitadd() { @@ -357,9 +366,9 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp) - - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + } summaryappend("\n") } From 31012ff5189f729a1f6c9b349ff5fc4aa0f61c13 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:51:29 +0100 Subject: [PATCH 174/177] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1898594..10fc165 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -117,7 +117,8 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true summaryappend("Does not exist: ${artifactfilter}," - } else { + } + if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { From 6539223840f6c7c0a0ec7bb2f3f8977760515a3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:53:31 +0100 Subject: [PATCH 175/177] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 10fc165..c1c4277 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -116,7 +116,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true - summaryappend("Does not exist: ${artifactfilter}," + summaryappend("Does not exist: ${artifactfilter},") } if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position From 93b2b46cd0c2a03fb0dbbd7311fdaa56c0facfdd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 14:09:53 +0100 Subject: [PATCH 176/177] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index c1c4277..0e7809e 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -95,7 +95,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - bool exceptionduringcopyartifacts = false + boolean exceptionduringcopyartifacts = false for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -116,7 +116,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true - summaryappend("Does not exist: ${artifactfilter},") + summaryappend("Does not exist:") } if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position From e9ce58fc62c3201c0186d528c66e077ce193c4c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 14:33:01 +0100 Subject: [PATCH 177/177] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0e7809e..1b21bcf 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -95,7 +95,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - boolean exceptionduringcopyartifacts = false + def boolean result = true for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -115,10 +115,10 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde ) } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" - exceptionduringcopyartifacts = true + result = false summaryappend("Does not exist:") } - if (!exceptionduringcopyartifacts) { + if (result) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { @@ -127,7 +127,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } summaryappend("${artifactfilter},") } - return exceptionduringcopyartifacts + return result } def releasegitadd() {