From 2f5617caea148498daadd3f4faf6aca95727777f Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Thu, 29 Nov 2018 15:13:15 +0000 Subject: [PATCH] Merge commit '5bb08476af818e51c87b40798049bf249bc2b6a4' into 1.10003.j20-comscr # Conflicts: # autobuild/Common_scripts/versionnumbering/metaversion.groovy --- artifacts/metaexport.groovy | 8 +- autobuilder.groovy | 10 ++- differences/do.groovy | 6 +- differences/show_release_differences.py | 20 +++-- import/importer.groovy | 4 +- meta/folder.groovy | 103 +++++++++++++++++++----- versionnumbering/generate.groovy | 17 ++-- versionnumbering/metaversion.groovy | 100 ----------------------- 8 files changed, 122 insertions(+), 146 deletions(-) delete mode 100644 versionnumbering/metaversion.groovy diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b6a832b..b4d4a6d 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,6 +4,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() + int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -13,11 +14,14 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + revisiontotal += importedmeta.buildstampinfo.revision artifactFolders.add("${destFolder}/**") } } - return artifactFolders; + def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) + return [artifactFolders, revisiontotal] } return this; diff --git a/autobuilder.groovy b/autobuilder.groovy index 2a04925..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -25,17 +27,21 @@ def getautobuildpath() { return autobuildroot } +def getmetafolderscript() { + return metafolderscript +} + // Start up boilderplate def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - def meta = metafolder.metareset(script) + def meta = getmetafolderscript().metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) diff --git a/differences/do.groovy b/differences/do.groovy index 35e0980..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,16 +1,16 @@ def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { def String cs = autobuilder.commonscripts() - def String difffolder = meta.folder() + def String difffile = meta.difffilepath() def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0813356..80b636e 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,10 +29,6 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(difffolder): - print("Creating folder (if not already there): {0}".format(difffolder)) - os.makedirs(difffolder, exist_ok = True) - def printDivider(outfh, times = 1, length = 100): for _ in range(times): outfh.write("{0}\n".format("*" * length)) @@ -84,6 +80,9 @@ jiraprojectpattern = { 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), + 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") @@ -172,7 +171,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") + parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +182,13 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - difffolder = args.difffolder + difffile = args.difffile print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - if difffolder: - makeFolder(difffolder) - outfh = open(os.path.join(difffolder, "differences.txt"), 'w') + if difffile: + outfh = open(difffile, 'w') else: outfh = sys.stdout @@ -209,7 +207,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if difffolder: + if difffile: outfh.close() return 0 except Exception as e: diff --git a/import/importer.groovy b/import/importer.groovy index a9e28b2..4d24d30 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -7,7 +7,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +21,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f26265..9c5367e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,8 +19,8 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - private String productname - private String subcomponentname + String productname + String subcomponentname String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { @@ -137,30 +137,40 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix - private int revision + int revision String buildstamp - String buildstampcommasep - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } - void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep + } + + String buildstampcommasep() { + def String buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + + void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") + revision = newrevision + def List buildstampsplit = buildstamp.split('\\.') + buildstampsplit[-1] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + script.echo("Build stamp now ${buildstamp}") } void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] - buildstampcommasep = rawmeta['buildstampcommasep'] } LinkedHashMap serialisationdata() { @@ -168,15 +178,14 @@ class BuildStampInfo extends GISJsonSerialised { 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, - 'buildstampcommasep' : buildstampcommasep, ] return data } } class ReleaseInfo extends GISJsonSerialised { - private String releaserepoURL - private ArrayList releaseitems + String releaserepoURL + ArrayList releaseitems ReleaseInfo(script, releaserepoURL = "") { super(script) @@ -376,10 +385,11 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private String FORMAT = "0.1" - private String METAFOLDERNAME = "meta" - private String METAFILENAME = "info.json" - private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String FORMAT = "0.1" + private static String METAFOLDERNAME = "meta" + private static String METAFILENAME = "info.json" + private static String METADIFFFILENAME = "differences.txt" + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -507,10 +517,14 @@ class MetaInfo extends GISJsonSerialised { return path } - String metafilepath() { + static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } - + + static String difffilepath() { + return METAFOLDERNAME + "/" + METADIFFFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) @@ -523,6 +537,17 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) + def importedmetadifffile = adjustedpath + "/" + difffilepath() + if (script.fileExists(importedmetadifffile)) { + def String newcontents = script.readFile(file: importedmetadifffile) + def String currentcontents = "" + if (script.fileExists(difffilepath())) { + currentcontents = script.readFile(file: difffilepath()) + } + + script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) + } + return mi } @@ -563,6 +588,42 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } + + // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java + private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { + if (maxdepth > 0) { + if (releaseinfo.releaserepoURL) { + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + releasemetacollection.add(rmi) + } + if (pathsofar) { + pathsofar += "/" + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + relatedmetainfo.offsetpath + releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) + } + } + return releasemetacollection + } + + ArrayList releasewalker(int maxdepth, String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + return releasemetacollection + } + + Map releasewalkerunique(int maxdepth, String startpath = "") { + Map releasemetauniquecollection = new LinkedHashMap() + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + for (relatedmetainfo in releasemetacollection) { + String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetauniquecollection + } } MetaInfo metareset(script) { @@ -575,4 +636,10 @@ MetaInfo metareset(script) { return mi } +MetaInfo metafromfolder(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 59fe2a3..a7b583a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,15 +1,18 @@ +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} + def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount def String gitrevision def String remoteinfo def String remoteurl dir( gitrootpath) { - def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -18,7 +21,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] @@ -28,8 +31,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { - def int MAXVERSIONNUMBER = 65535 - echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -65,7 +66,7 @@ def getVersionPrefix(String sourcebranch) { def integers = [] for (integerstring in integerstrings) { int integer = integerstring.toInteger() - integer = integer % (MAXVERSIONNUMBER + 1) + integer = integer % (maxversionnumber() + 1) integers.add(integer) } @@ -112,7 +113,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum echo "Build stamp is: ${buildstamp}" meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) return buildstamp } diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy deleted file mode 100644 index 44bfbde..0000000 --- a/versionnumbering/metaversion.groovy +++ /dev/null @@ -1,100 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - -def prefixline(rootpath, filepath) { - def fileContents = readFile("${filepath}").trim() - def fileContentsNormalized = fileContents.normalize() - - def newFileContents = "" - def lines = fileContentsNormalized.readLines() - for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + "${nl}" - } - return newFileContents -} - -def mergemeta(rootpath, versionFolder, outputFolder) { - echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents - - def intrevision = 0 - if (fileExists("${versionFolder}/revision.txt")) { - def revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - } - else { - intrevision = 1 - } - - return intrevision -} - -def appendcomponentinfo(rootpath, versionFolder, outputFolder) { - echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents - } - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = readFile("${buildStampFile}").trim() - - def remoteurlFile = "${versionFolder}/remoteurl.txt" - def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - - def gitRevisionFile = "${versionFolder}/gitrevision.txt" - def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - def releaseJobNameFile = "${versionFolder}/releasejobname.txt" - def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - def revision = readFile("${versionFolder}/revision.txt").trim() - def intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - - return intrevision -} - -// Each version folder should have been created using version creation scripts in this Common_scripts folder -// as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - def totalrevision = 0 - dir("${outputFolder}") { - deleteDir() - } - for (artifactItem in artifactList) { - (rootpath, versionFolder) = artifactItem - echo "Inspecting folder: ${versionFolder}" - def metaversionsignaturefile = "${versionFolder}/metaversion.txt" - // If the sub project is a meta job - def intrevision = 0 - if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta(rootpath, versionFolder, outputFolder) - } - else { - intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) - } - totalrevision += intrevision - } - echo "Accumulated revision is: ${totalrevision}" - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) -} - -return this; \ No newline at end of file