diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 043c4de..67d13c3 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,8 +1,6 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(meta, subJobs) { - def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - +def combineSubjobArtifacts(meta, importer, subJobs) { def ArrayList artifactFolders = new ArrayList() for (subJob in subJobs) { def subJobObject = subJob[0] diff --git a/autobuilder.groovy b/autobuilder.groovy index 63ca2de..1256394 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,9 +1,7 @@ // Assume that WORKSPACE environment variable is set autobuildroot = "${WORKSPACE}/autobuild" - commonscriptsroot = "${autobuildroot}/Common_scripts" - metafolderscript = null def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { @@ -32,7 +30,7 @@ def getmetafolderscript() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -42,13 +40,13 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set(productname, subcomponentname, jirakey) + meta.productinfo.set(productname, subcomponentname) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def rr = load("${commonscripts()}/release/registerrepo.groovy") - rr.registerrepo(meta, releaserepo, repoRootSuffix) + rr.registerrepo(meta, releaserepo) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) @@ -56,6 +54,11 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA return meta } +def start(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String jirakey = JIRA_KEY, String releaserepo = RELEASE_REPO, String repoRootSuffix = "", String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { + echo "DEPRECATED method: start(script, String productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, ...)" + return startbuild(script, productname, subcomponentname, releaserepo, autobuildpath, commonscriptspath) +} + // Start up boilderplate def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting non-build job...." @@ -67,7 +70,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") + meta.productinfo.set("Non-build-product", "Non-build-subcomponent") def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) diff --git a/copy/esw.groovy b/copy/esw.groovy new file mode 100644 index 0000000..2a63171 --- /dev/null +++ b/copy/esw.groovy @@ -0,0 +1,63 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + env.ESW = 'esw' + env.ESWDEST = "${WORKSPACE}/${ESW}" + env.ESWEXPORT = 'embedded_software' + cleanfolders = [ + ESWDEST, + ESWEXPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying embedded software artifacts') { + steps { + script { + importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) + sh """ + rm -v -rf diff + rm -v -rf meta + mv -v "${ESW}"/* . + """ + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} \ No newline at end of file diff --git a/copy/logic.groovy b/copy/logic.groovy index 4b3d521..d1ece6d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -5,16 +5,18 @@ pipeline { steps { script { meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") } } } stage('Clean exports') { steps { script { + env.LOGICDEST = "${WORKSPACE}/logic" cleanfolders = [ - "${WORKSPACE}/logic", + LOGICDEST, "${WORKSPACE}/embedded_logic", - "${WORKSPACE}/diff", // Some old artifacts have this folder ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -24,11 +26,7 @@ pipeline { stage('Copying logic artifacts') { steps { script { - env.LOGICDEST = "${WORKSPACE}/logic" - if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) - } + importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC_RELEASE_REVISION, LOGICDEST) } } } @@ -37,8 +35,9 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf diff rm -v -rf meta + # The following is removing a deprecated folder + rm -v -rf diff mv -v "logic"/* . """ archiveArtifacts "${artifactlist}" diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 2e94820..6280960 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -29,24 +29,24 @@ class JiraTicketInfo { class DifferenceRecord { String productname String subcomponentname - String jirakey String rawdifferencetext String sha String type String mergeinfo + String squashinfo List jiraticketinfo String comment DifferenceRecord() { productname = "" subcomponentname = "" - jirakey = "" rawdifferencetext = "" sha = "" type = "" mergeinfo = "" + squashinfo = "" jiraticketinfo = [] comment = "" } @@ -56,7 +56,7 @@ class DifferenceRecord { for (j in jiraticketinfo) { jiraticketstring += j.dump() + " " } - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -140,7 +140,6 @@ def String getcomponent(meta) { } def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { - def differencerecords = [] def String releaseexpression if (tripledot) { releaseexpression = earlyrevision + "..." + laterrevision @@ -167,6 +166,16 @@ def extractsha(String line) { return [sha.trim(), restofline.trim()] } +def extractsquashes(String line) { + def pattern1 = "^(.*)(Squashed .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + return [false, line] +} + def extractmerges(String line) { def pattern1 = "^(.*)(Merge branch .*)" def match1 = (line =~ pattern1) @@ -303,12 +312,39 @@ def extractreverts(String line) { return [false, [], line] } -def DifferenceRecord fillrecord(meta, line) { +def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() + } + return differencetext +} + +def List fillsquashrecords(String sourcepath, String sha) { + def List differencerecords = [] + + String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) + int count = 0 + squashloglines.split('\n').each { + if (count > 0) { // Ignore first line + def line = it + if (line?.trim()) { + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) + //echo "${dr.dump()}" + } + } + count += 1 + } + return differencerecords +} + +def List fillrecord(meta, line, String sourcepath) { + def List differencerecords = [] def DifferenceRecord dr = new DifferenceRecord() dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname - dr.jirakey = meta.productinfo.jirakey dr.rawdifferencetext = line @@ -317,41 +353,49 @@ def DifferenceRecord fillrecord(meta, line) { (sha, line2) = extractsha(line) dr.sha = sha - - (result, mergeinfo) = extractmerges(line2) + + (result, squashinfo) = extractsquashes(line2) if (result) { - dr.type = "Merge" - dr.mergeinfo = mergeinfo + dr.type = "Squash" + dr.squashinfo = squashinfo + squashrecords = fillsquashrecords(sourcepath, sha) + differencerecords.addAll(squashrecords) } else { - (result, jiraticketinfo, comment) = extractreverts(line2) + (result, mergeinfo) = extractmerges(line2) if (result) { - dr.type = "Revert" - dr.jiraticketinfo = jiraticketinfo - dr.comment = comment + dr.type = "Merge" + dr.mergeinfo = mergeinfo } else { - (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + (result, jiraticketinfo, comment) = extractreverts(line2) if (result) { - dr.type = "Ticketed" + dr.type = "Revert" dr.jiraticketinfo = jiraticketinfo dr.comment = comment } else { - dr.type = "Unticketed" - dr.comment = line2 + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.comment = line2 + } } } } - - return dr + differencerecords.add(dr) + return differencerecords } -def List processtext(meta, String text) { +def List processtext(meta, String text, String sourcepath) { def List differencerecords = [] text.split('\n').each { def line = it if (line?.trim()) { - def DifferenceRecord dr = fillrecord(meta, line.trim()) - differencerecords.add(dr) + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) //echo "${dr.dump()}" } } @@ -371,7 +415,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - differencerecords = processtext(earlymeta, differencetext) + differencerecords = processtext(earlymeta, differencetext, sourcepath) if (earlyrevision == laterrevision) { summaryappend(" No change") } @@ -395,6 +439,28 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } +class SquashReportRecord { + String comment + int duplicates + + SquashReportRecord() { + comment = '' + duplicates = 0 + } +} + +def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { + dir(reportsubfolder) { + def String squashlist = "${title}\nSquash list:\n" + def squashessorted = squashes.sort() + for (asquash in squashessorted) { + def SquashReportRecord s = asquash.value + squashlist += "${s.comment} (${s.duplicates})\n" + } + writeFile(file: "squashlist.txt", text: squashlist) + } +} + class MergeReportRecord { String comment int duplicates @@ -547,12 +613,25 @@ def differencerecordsreport(String reportsubfolder, String title, differencereco def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] + def Map squashes = [:] def Map merges = [:] def Map reverts = [:] def Map unticketed = [:] def Map tickets = [:] for (dr in differencerecords) { + if (dr.type == "Squash") { + def SquashReportRecord s + if (!squashes.containsKey(dr.mergeinfo)) { + s = new SquashReportRecord() + s.comment = dr.squashinfo + s.duplicates = 1 + } else { + s = squashes[dr.squashinfo] + s.duplicates += 1 + } + squashes[dr.squashinfo] = s + } if (dr.type == "Merge") { def MergeReportRecord m if (!merges.containsKey(dr.mergeinfo)) { @@ -665,6 +744,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) { reportsubfolder = "${reportfolder}/Combined" } mergereport(reportsubfolder, title, merges, iscomponent) + squashreport(reportsubfolder, title, squashes, iscomponent) revertsreport(reportsubfolder, title, reverts, iscomponent) ticketedreport(reportsubfolder, title, ticketed, iscomponent) unticketedreport(reportsubfolder, title, unticketed, iscomponent) @@ -720,8 +800,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - def earlymeta = meta.importrelated(earlymetafolder) - def latermeta = meta.importrelated(latermetafolder) + def earlymeta = meta.importrelated(true, earlymetafolder) + def latermeta = meta.importrelated(true, latermetafolder) def earlymetaitems = earlymeta.releasewalkerunique(md) def latermetaitems = latermeta.releasewalkerunique(md) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py deleted file mode 100644 index 5c0f15d..0000000 --- a/differences/show_release_differences.py +++ /dev/null @@ -1,225 +0,0 @@ -''' -showreleasedifferences -- shortdesc - -showreleasedifferences generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import subprocess -import os -import sys -import re - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") - -class GitException(Exception): - def __init__(self, message): - self.message = message - - def __repr__(self): - return self.message - -def printDivider(outfh, times = 1, length = 100): - for _ in range(times): - outfh.write("{0}\n".format("*" * length)) - -TABSIZE = 4 -def printTabbed(outfh, string, tabs = 0): - outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) - -def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): - if tripledot: - releaseExpression = previousRelease + "..." + currentRelease - else: - releaseExpression = previousRelease + ".." + currentRelease - currentdir = os.getcwd() - os.chdir(gitrootfolder) - gitlog = [] - printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) - proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) - while True: - line = proc.stdout.readline() - if line: - linestring = line.decode('utf-8').rstrip() - gitlog.append(linestring) - #printTabbed(outfh, linestring, 1) - else: - break - proc.wait() - if proc.returncode > 0: - raise GitException("Git executable returned an error") - os.chdir(currentdir) - return gitlog - -jiraprojectpattern = { - 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), - ), - 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), - ), - 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), - ), - 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), - ), - 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), - ), - 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), - ), - 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), - re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), - ), - } - -mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") - -def processLog(gitlog, jirakeyprefix): - reverts = [] - merges = {} - other = [] - numbered = {} - (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] - for line in gitlog: - expectedmatch = expectedpattern.match(line) - revertmatch = revertpattern.match(line) - mergematch = mergepattern.match(line) - if expectedmatch == None and revertmatch == None and mergematch == None: - other.append(line) - if revertmatch: - gisnumber = jirakeyprefix + "-" + revertmatch.group(2) - description = revertmatch.group(3) - reverts.append("Revert: {0} {1}".format(gisnumber, description)) - if mergematch: - merge = mergematch.group(2) - if not merge in merges: - merges[merge] = 1 - else: - merges[merge] += 1 - if expectedmatch: - gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) - description = expectedmatch.group(2) - if not gisnumber in numbered: - numbered[gisnumber] = [] - numbered[gisnumber].append(description) - return (reverts, merges, other, numbered) - -def showDifferences(differences, jirakeyprefix, outfh): - (revertslog, mergeslog, otherlog, numberedlog) = differences - printDivider(outfh) - printTabbed(outfh, jirakeyprefix + "-Numbers Log:") - for (gisnumber, descriptions) in sorted(numberedlog.items()): - printTabbed(outfh, gisnumber, 1) - descriptions = numberedlog[gisnumber]; - for description in descriptions: - printTabbed(outfh, description, 2) - printDivider(outfh) - printTabbed(outfh, "Revert log:") - for reversion in sorted(revertslog): - printTabbed(outfh, reversion, 1) - printDivider(outfh) - printTabbed(outfh, "Merges log:") - for merge in sorted(mergeslog): - printTabbed(outfh, merge, 1) - printDivider(outfh) - printTabbed(outfh, "Other log:") - for other in sorted(otherlog): - printTabbed(outfh, other, 1) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") - parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") - parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") - parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - gitrootfolder = args.gitrootfolder - sourceversion = args.sourceversion - previousversion = args.previousversion - jirakeyprefix = args.jirakeyprefix - outputoptions = args.outputoptions - difffile = args.difffile - - print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - - if difffile: - os.makedirs(os.path.dirname(difffile), exist_ok=True) - outfh = open(difffile, 'w') - else: - outfh = sys.stdout - - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if 'opposite' in outputoptions.lower(): - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if 'notcommon' in outputoptions.lower(): - printDivider(outfh, times = 4) - printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) - gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) - showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - - if difffile: - outfh.close() - return 0 - except Exception as e: - print("Error while determining differences:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/import/importer.groovy b/import/importer.groovy index 25b2614..41f5118 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,36 +1,122 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { - copyArtifacts( - filter: filter, - excludes: excludes, - fingerprintArtifacts: fingerprint, - projectName: jenkinsjob.trim(), - selector: specific(buildnumber.toString()), - target: destpath - ) - def importedmeta = meta.importrelated(destpath) - if (incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) +class ImportOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String m_filter // Only import files/paths/folders that match the filter + String m_excludes // Exclude files/paths/folders that match the exclude filter + boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_appendtoparentmeta // Append to parent meta + String m_sourceselect // Where are the artifacts coming from + + String m_jobname + def m_buildnumberorselector + + String m_repo + String m_revision + + ImportOptions(script, + String sourceselect = 'Jenkins job artifact', + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false, + appendtoparentmeta = true) { + m_script = script + m_sourceselect = sourceselect.trim() + m_incrementversionnumber = incrementversionnumber + m_filter = filter.trim() + m_excludes = excludes.trim() + m_fingerprint = fingerprint + m_nested = nested + m_appendtoparentmeta = appendtoparentmeta + } + + void setsourceselect(String sourceselect) { + sourceselect = sourceselect.trim() + switch (sourceselect) { + case 'Jenkins job artefact': + m_sourceselect = 'Jenkins job artifact' + break + case 'Jenkins job artifact': + case 'Release repository': + m_sourceselect = sourceselect + break + default: + error("Unknown type of artifact source select: ${m_sourceselect}") + } + } + + void setincrementversionnumber(boolean incrementversionnumber) { + m_incrementversionnumber = incrementversionnumber + } + + void setfilter(String filter) { + m_filter = filter.trim() + } + + void setexcludes(String excludes) { + m_excludes = excludes.trim() + } + + void setfingerprint(boolean fingerprint) { + m_fingerprint = fingerprint + } + + void setnested(boolean nested) { + m_nested = nested + } + + void setappendtoparentmeta(boolean appendtoparentmeta) { + m_appendtoparentmeta = appendtoparentmeta + } + + void jenkinsjob(String jobname, String buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = buildnumberorselector.trim() + setsourceselect('Jenkins job artifact') + } + + void releaserepo(String repo, String revision) { + m_repo = repo.trim() + m_revision = revision.trim() + setsourceselect('Release repository') + } + + void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) { + jenkinsjob(jobname, buildnumberorselector) + releaserepo(repo, revision) + setsourceselect(sourceselect) } - return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { +ImportOptions getNewImportOptions() { + return new ImportOptions(this) +} + +def artifactimportfrombuildjob(meta, importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Jenkins job artifact') { + error("Import options settings have not been set for: Jenkins job artifact") + return + } + copyArtifacts( - filter: filter, - excludes: excludes, - fingerprintArtifacts: fingerprint, - projectName: jenkinsjob.trim(), + filter: importoptions.m_filter, + excludes: importoptions.m_excludes, + fingerprintArtifacts: importoptions.m_fingerprint, + projectName: importoptions.m_jobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: buildselector + parameterName: importoptions.m_buildnumberorselector ], target: destpath ) - def importedmeta = meta.importrelated(destpath) - if (incrementversionnumber) { + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) + if (importoptions.m_incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta @@ -42,16 +128,20 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") -def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { - def rooturl = repo.trim() - def rootrevision = revision.trim() +def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) relatedimportinfo = importedmeta.getassociated() echo "Cloning associated repos" @@ -76,46 +166,120 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - if (incrementversionnumber) { + if (importoptions.m_incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { - def rooturl = repo.trim() - def rootrevision = revision.trim() +def artifactimportfromrepo(meta, importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - if (incrementversionnumber) { + if (importoptions.m_incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { - switch (source) { - case "Jenkins job artifact": - return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) - case "Release repository": - if (nested) { - return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) +def artifactimport(meta, importoptions, String destpath) { + switch (importoptions.m_sourceselect) { + case 'Jenkins job artifact': + case 'Jenkins job artefact': + return artifactimportfrombuildjob(meta, importoptions, destpath) + case 'Release repository': + if (importoptions.m_nested) { + return hierarchyartifactimportfromrepo(meta, importoptions, destpath) } - return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + return artifactimportfromrepo(meta, importoptions, destpath) default: error("Unknown type of artifact source: ${source}") } return meta } +// Deprecated (use above) +def artifactimport(meta, + String sourceselect, + String jenkinsjob, + buildnumberorselector, + String repo, + String revision, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false) { + echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" + def importoptions = getNewImportOptions() + + importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + importoptions.setnested(nested) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + int buildnumber, + String destpath, + boolean incrementversionnumber = true, + String filter = '**', + String excludes = '', + boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" + def importoptions = getNewImportOptions() + + importoptions.jenkinsjob(buildjob, " ${buildnumber}") + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + buildnumberorselector, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" + def importoptions = getNewImportOptions() + + importoptions.jenkinsjob(buildjob, buildnumberorselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + return this; diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6fcdc6c..96a8b2e 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -23,10 +23,10 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { artifactFetched = [:] - def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + def importoptions = importer.getNewImportOptions() for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] if (artifactKeys instanceof java.lang.String) { @@ -35,7 +35,12 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { for (artifactKey in artifactKeys) { (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) + importoptions.jenkinsjob(sourceProject, sourceselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(toCopy) + importoptions.setexcludes(notToCopy) + + def importmeta = importer.artifactimport(meta, importoptions, destFolder) artifactFetched.put(artifactKey, importmeta) } } diff --git a/meta/folder.groovy b/meta/folder.groovy index 40f03aa..f600155 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -22,16 +22,14 @@ abstract class GISJsonSerialised implements Serializable { class ProductInfo extends GISJsonSerialised { String productname String subcomponentname - String jirakey - ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + ProductInfo(script, productname = "", subcomponentname = "") { super(script) this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jirakey } - void set(String productname, String subcomponentname, String jirakey) { + void set(String productname, String subcomponentname) { // Make sure the fields can be used in git tags def int pstatus = 0 if (script.isUnix()) { @@ -55,20 +53,17 @@ class ProductInfo extends GISJsonSerialised { } this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jirakey } void deserialiseJson(rawmeta) { productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] - jirakey = rawmeta['jirakey'] } LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, - 'jirakey' : jirakey, ] return data } @@ -162,7 +157,7 @@ class BuildStampInfo extends GISJsonSerialised { int revision String buildstamp - BuildStampInfo(script, buildstampprefix = "1.30000.0.", revision = 0, buildstamp = "1.30000.0.0") { + BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision @@ -605,7 +600,7 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME + "/" + METAFILENAME } - MetaInfo importrelated(String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { + MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { MetaInfo mi = new MetaInfo(script) def String adjustedpath = relativize(destpath) @@ -618,7 +613,9 @@ class MetaInfo extends GISJsonSerialised { } RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) - associatedmeta.add(rmi) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } return mi } diff --git a/release/do.groovy b/release/do.groovy index a9f0a3e..817fd5f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -4,6 +4,8 @@ unstablereason = "" summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" +repoandtagsfile = "${summaryfolder}/repoandtags.txt" +repoandtagsstring = "" releaserepofolder = "${WORKSPACE}/releaserepos" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" @@ -28,6 +30,8 @@ def preconditions() { unstablereason = "" summarystring = "" writeFile(file: summaryfile, text: "Release summary: \n") + repoandtagsstring = "" + writeFile(file: repoandtagsfile, text: "Repository summary: \n") } def setunstable(String reason) { @@ -42,6 +46,12 @@ def summaryappend(String summary) { summarystring += summary } +def repoandtagsappend(String summary) { + def String repoandtagscontents = readFile(file: repoandtagsfile) + writeFile(file: repoandtagsfile, text: repoandtagscontents + summary) + repoandtagsstring += summary +} + def getpreviousreleasemeta(buildmeta, String checkoutfolder) { def previousreleasemeta = buildmeta.getNewMetaObject(this) // See if there is meta data file from the current release @@ -55,6 +65,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) { def checkoutreleaserepo(buildmeta, String checkoutfolder) { def String releaserepo = buildmeta.releaseinfo.releaserepoURL // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + repoandtagsappend(" Release repo: ${releaserepo}") def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { sh("git clone ${releaserepo} ${checkoutfolder}") @@ -70,6 +81,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl if (sourcerepo) { + repoandtagsappend(" Source repo: ${sourcerepo}") // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component if (fileExists(checkoutfolder)) { dir(checkoutfolder) { @@ -89,6 +101,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { sh("git clone ${sourcerepo} ${checkoutfolder}") } } + } else { + repoandtagsappend(" Source repo: None") } } @@ -365,6 +379,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) @@ -374,6 +389,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri } summaryappend("\n") + repoandtagsappend("\n") } def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index 93e4711..865e5fa 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -66,7 +66,7 @@ pipeline { echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" if (isUnix()) { sh """#!/bin/bash - find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \; + find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \\; # Uses mono "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml """ diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 751d1c8..f5b2ebd 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -80,7 +80,8 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status" + echo "Found ${gitURL} checked out at ${reporootsubfolder}" +/* dir(reporootsubfolder) { if (isUnix()) { sh """ @@ -94,6 +95,7 @@ def String checkouttoreporoot(String gitURL) { """ } } +*/ } } return reporootsubfolder diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index de60b66..71d7bf6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -67,14 +67,14 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() - def defaultversionprefix = "1.30000.0." + def defaultversionprefix = "0.30000.0." def splits = sourcebranch.split('/') if (splits.size() < 1) { echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" return defaultversionprefix } - def mastertypeversionprefix = "1.20000.0." + def mastertypeversionprefix = "0.20000.0." sourcebranch = splits.last() if (sourcebranch.toLowerCase().find("master") != null) { echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" @@ -83,7 +83,7 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { def splits2 = sourcebranch.split('\\.') if (splits2.size() < 1) { - def thirdpartyversionprefix = "1.25000.0." + def thirdpartyversionprefix = "0.25000.0." echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" return thirdpartyversionprefix } diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 1a65e6d..441ad06 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -8,7 +8,7 @@ def main(argv): # Call once arguments to allow or disallow something parser.add_argument('-f', '--filetoconvert', help='The file to convert') - parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace') parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') args = parser.parse_args(argv)