This commit is contained in:
Dan Hawkins 2020-01-15 10:17:51 +00:00
commit 783bf697a5
14 changed files with 431 additions and 329 deletions

View file

@ -1,8 +1,6 @@
// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders
def combineSubjobArtifacts(meta, subJobs) {
def importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
def combineSubjobArtifacts(meta, importer, subJobs) {
def ArrayList<String> artifactFolders = new ArrayList<String>()
for (subJob in subJobs) {
def subJobObject = subJob[0]

View file

@ -1,9 +1,7 @@
// Assume that WORKSPACE environment variable is set
autobuildroot = "${WORKSPACE}/autobuild"
commonscriptsroot = "${autobuildroot}/Common_scripts"
metafolderscript = null
def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") {
@ -32,7 +30,7 @@ def getmetafolderscript() {
}
// Start up boilderplate
def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) {
def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) {
echo "Starting...."
autobuildroot = autobuildpath
@ -42,13 +40,13 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA
conditionalclean()
def meta = getmetafolderscript().metareset(script)
meta.productinfo.set(productname, subcomponentname, jirakey)
meta.productinfo.set(productname, subcomponentname)
def now = new Date()
now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC'))
meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString())
def rr = load("${commonscripts()}/release/registerrepo.groovy")
rr.registerrepo(meta, releaserepo, repoRootSuffix)
rr.registerrepo(meta, releaserepo)
def ch = load("${commonscripts()}/configuration/configuration.groovy")
ch.registerPipelineScriptConfiguration(meta)
@ -56,6 +54,11 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA
return meta
}
def start(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String jirakey = JIRA_KEY, String releaserepo = RELEASE_REPO, String repoRootSuffix = "", String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) {
echo "DEPRECATED method: start(script, String productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, ...)"
return startbuild(script, productname, subcomponentname, releaserepo, autobuildpath, commonscriptspath)
}
// Start up boilderplate
def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) {
echo "Starting non-build job...."
@ -67,7 +70,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com
conditionalclean()
def meta = getmetafolderscript().metareset(script)
meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key")
meta.productinfo.set("Non-build-product", "Non-build-subcomponent")
def now = new Date()
now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC'))
meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString())

63
copy/esw.groovy Normal file
View file

@ -0,0 +1,63 @@
pipeline {
agent { label "${AGENT_LABEL}" }
stages {
stage('Initial setup') {
steps {
script {
meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this)
importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
}
}
}
stage('Clean exports') {
steps {
script {
env.ESW = 'esw'
env.ESWDEST = "${WORKSPACE}/${ESW}"
env.ESWEXPORT = 'embedded_software'
cleanfolders = [
ESWDEST,
ESWEXPORT,
]
cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy")
cleaner.clean(cleanfolders)
}
}
}
stage('Copying embedded software artifacts') {
steps {
script {
importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST)
}
}
}
stage('Archiving artifacts') {
steps {
script {
def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"])
sh """
rm -v -rf diff
rm -v -rf meta
mv -v "${ESW}"/* .
"""
archiveArtifacts "${artifactlist}"
}
}
}
}
post {
always {
echo 'Post-build'
}
failure {
echo 'Failed build'
}
unstable {
echo 'Unstable build'
}
success {
echo 'Succeeded build'
}
}
}

View file

@ -5,16 +5,18 @@ pipeline {
steps {
script {
meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this)
importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
}
}
}
stage('Clean exports') {
steps {
script {
env.LOGICDEST = "${WORKSPACE}/logic"
cleanfolders = [
"${WORKSPACE}/logic",
LOGICDEST,
"${WORKSPACE}/embedded_logic",
"${WORKSPACE}/diff", // Some old artifacts have this folder
]
cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy")
cleaner.clean(cleanfolders)
@ -24,11 +26,7 @@ pipeline {
stage('Copying logic artifacts') {
steps {
script {
env.LOGICDEST = "${WORKSPACE}/logic"
if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") {
importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST)
}
importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC_RELEASE_REVISION, LOGICDEST)
}
}
}
@ -37,8 +35,9 @@ pipeline {
script {
def artifactlist = meta.generateartifacts(["embedded_logic/**"])
sh """
rm -v -rf diff
rm -v -rf meta
# The following is removing a deprecated folder
rm -v -rf diff
mv -v "logic"/* .
"""
archiveArtifacts "${artifactlist}"

View file

@ -29,24 +29,24 @@ class JiraTicketInfo {
class DifferenceRecord {
String productname
String subcomponentname
String jirakey
String rawdifferencetext
String sha
String type
String mergeinfo
String squashinfo
List<JiraTicketInfo> jiraticketinfo
String comment
DifferenceRecord() {
productname = ""
subcomponentname = ""
jirakey = ""
rawdifferencetext = ""
sha = ""
type = ""
mergeinfo = ""
squashinfo = ""
jiraticketinfo = []
comment = ""
}
@ -56,7 +56,7 @@ class DifferenceRecord {
for (j in jiraticketinfo) {
jiraticketstring += j.dump() + " "
}
return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}"
return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}"
}
}
@ -140,7 +140,6 @@ def String getcomponent(meta) {
}
def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) {
def differencerecords = []
def String releaseexpression
if (tripledot) {
releaseexpression = earlyrevision + "..." + laterrevision
@ -167,6 +166,16 @@ def extractsha(String line) {
return [sha.trim(), restofline.trim()]
}
def extractsquashes(String line) {
def pattern1 = "^(.*)(Squashed .*)"
def match1 = (line =~ pattern1)
if (match1.find()) {
return [true, match1.group(2)]
}
return [false, line]
}
def extractmerges(String line) {
def pattern1 = "^(.*)(Merge branch .*)"
def match1 = (line =~ pattern1)
@ -303,12 +312,39 @@ def extractreverts(String line) {
return [false, [], line]
}
def DifferenceRecord fillrecord(meta, line) {
def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) {
def String differencetext
dir(gitrootfolder) {
differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim()
}
return differencetext
}
def List fillsquashrecords(String sourcepath, String sha) {
def List differencerecords = []
String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha)
int count = 0
squashloglines.split('\n').each {
if (count > 0) { // Ignore first line
def line = it
if (line?.trim()) {
def List drs = fillrecord(meta, line.trim(), sourcepath)
differencerecords.addAll(drs)
//echo "${dr.dump()}"
}
}
count += 1
}
return differencerecords
}
def List fillrecord(meta, line, String sourcepath) {
def List differencerecords = []
def DifferenceRecord dr = new DifferenceRecord()
dr.productname = meta.productinfo.productname
dr.subcomponentname = meta.productinfo.subcomponentname
dr.jirakey = meta.productinfo.jirakey
dr.rawdifferencetext = line
@ -317,41 +353,49 @@ def DifferenceRecord fillrecord(meta, line) {
(sha, line2) = extractsha(line)
dr.sha = sha
(result, mergeinfo) = extractmerges(line2)
(result, squashinfo) = extractsquashes(line2)
if (result) {
dr.type = "Merge"
dr.mergeinfo = mergeinfo
dr.type = "Squash"
dr.squashinfo = squashinfo
squashrecords = fillsquashrecords(sourcepath, sha)
differencerecords.addAll(squashrecords)
} else {
(result, jiraticketinfo, comment) = extractreverts(line2)
(result, mergeinfo) = extractmerges(line2)
if (result) {
dr.type = "Revert"
dr.jiraticketinfo = jiraticketinfo
dr.comment = comment
dr.type = "Merge"
dr.mergeinfo = mergeinfo
} else {
(result, jiraticketinfo, comment) = extractjiraticketinfo(line2)
(result, jiraticketinfo, comment) = extractreverts(line2)
if (result) {
dr.type = "Ticketed"
dr.type = "Revert"
dr.jiraticketinfo = jiraticketinfo
dr.comment = comment
} else {
dr.type = "Unticketed"
dr.comment = line2
(result, jiraticketinfo, comment) = extractjiraticketinfo(line2)
if (result) {
dr.type = "Ticketed"
dr.jiraticketinfo = jiraticketinfo
dr.comment = comment
} else {
dr.type = "Unticketed"
dr.comment = line2
}
}
}
}
return dr
differencerecords.add(dr)
return differencerecords
}
def List processtext(meta, String text) {
def List processtext(meta, String text, String sourcepath) {
def List differencerecords = []
text.split('\n').each {
def line = it
if (line?.trim()) {
def DifferenceRecord dr = fillrecord(meta, line.trim())
differencerecords.add(dr)
def List drs = fillrecord(meta, line.trim(), sourcepath)
differencerecords.addAll(drs)
//echo "${dr.dump()}"
}
}
@ -371,7 +415,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) {
echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}"
summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}")
def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision)
differencerecords = processtext(earlymeta, differencetext)
differencerecords = processtext(earlymeta, differencetext, sourcepath)
if (earlyrevision == laterrevision) {
summaryappend(" No change")
}
@ -395,6 +439,28 @@ def addedcomponent(latermetaitems, key) {
summaryappend("Added ")
}
class SquashReportRecord {
String comment
int duplicates
SquashReportRecord() {
comment = ''
duplicates = 0
}
}
def squashreport(String reportsubfolder, String title, LinkedHashMap<String, SquashReportRecord> squashes, boolean iscomponent = true) {
dir(reportsubfolder) {
def String squashlist = "${title}\nSquash list:\n"
def squashessorted = squashes.sort()
for (asquash in squashessorted) {
def SquashReportRecord s = asquash.value
squashlist += "${s.comment} (${s.duplicates})\n"
}
writeFile(file: "squashlist.txt", text: squashlist)
}
}
class MergeReportRecord {
String comment
int duplicates
@ -547,12 +613,25 @@ def differencerecordsreport(String reportsubfolder, String title, differencereco
def boolean report(meta, title, differencerecords, boolean iscomponent = true) {
def Map ticketed = [:]
def Map squashes = [:]
def Map merges = [:]
def Map reverts = [:]
def Map unticketed = [:]
def Map tickets = [:]
for (dr in differencerecords) {
if (dr.type == "Squash") {
def SquashReportRecord s
if (!squashes.containsKey(dr.mergeinfo)) {
s = new SquashReportRecord()
s.comment = dr.squashinfo
s.duplicates = 1
} else {
s = squashes[dr.squashinfo]
s.duplicates += 1
}
squashes[dr.squashinfo] = s
}
if (dr.type == "Merge") {
def MergeReportRecord m
if (!merges.containsKey(dr.mergeinfo)) {
@ -665,6 +744,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) {
reportsubfolder = "${reportfolder}/Combined"
}
mergereport(reportsubfolder, title, merges, iscomponent)
squashreport(reportsubfolder, title, squashes, iscomponent)
revertsreport(reportsubfolder, title, reverts, iscomponent)
ticketedreport(reportsubfolder, title, ticketed, iscomponent)
unticketedreport(reportsubfolder, title, unticketed, iscomponent)
@ -720,8 +800,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin
echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}"
def earlymeta = meta.importrelated(earlymetafolder)
def latermeta = meta.importrelated(latermetafolder)
def earlymeta = meta.importrelated(true, earlymetafolder)
def latermeta = meta.importrelated(true, latermetafolder)
def earlymetaitems = earlymeta.releasewalkerunique(md)
def latermetaitems = latermeta.releasewalkerunique(md)

View file

@ -1,225 +0,0 @@
'''
showreleasedifferences -- shortdesc
showreleasedifferences generates a version folder with a common set of version information files in it
@author: dan
@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved.
'''
import subprocess
import os
import sys
import re
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
__all__ = []
__version__ = 0.1
__date__ = '2017-11-29'
__updated__ = '2017-11-29'
gitExecutable = os.getenv("GIT_EXECUTABLE", default="git")
class GitException(Exception):
def __init__(self, message):
self.message = message
def __repr__(self):
return self.message
def printDivider(outfh, times = 1, length = 100):
for _ in range(times):
outfh.write("{0}\n".format("*" * length))
TABSIZE = 4
def printTabbed(outfh, string, tabs = 0):
outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string))
def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False):
if tripledot:
releaseExpression = previousRelease + "..." + currentRelease
else:
releaseExpression = previousRelease + ".." + currentRelease
currentdir = os.getcwd()
os.chdir(gitrootfolder)
gitlog = []
printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression))
proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE)
while True:
line = proc.stdout.readline()
if line:
linestring = line.decode('utf-8').rstrip()
gitlog.append(linestring)
#printTabbed(outfh, linestring, 1)
else:
break
proc.wait()
if proc.returncode > 0:
raise GitException("Git executable returned an error")
os.chdir(currentdir)
return gitlog
jiraprojectpattern = {
'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'),
),
'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'),
),
'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'),
),
'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'),
),
'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'),
),
'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'),
),
'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'),
),
}
mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$")
def processLog(gitlog, jirakeyprefix):
reverts = []
merges = {}
other = []
numbered = {}
(expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix]
for line in gitlog:
expectedmatch = expectedpattern.match(line)
revertmatch = revertpattern.match(line)
mergematch = mergepattern.match(line)
if expectedmatch == None and revertmatch == None and mergematch == None:
other.append(line)
if revertmatch:
gisnumber = jirakeyprefix + "-" + revertmatch.group(2)
description = revertmatch.group(3)
reverts.append("Revert: {0} {1}".format(gisnumber, description))
if mergematch:
merge = mergematch.group(2)
if not merge in merges:
merges[merge] = 1
else:
merges[merge] += 1
if expectedmatch:
gisnumber = jirakeyprefix + "-" + expectedmatch.group(1)
description = expectedmatch.group(2)
if not gisnumber in numbered:
numbered[gisnumber] = []
numbered[gisnumber].append(description)
return (reverts, merges, other, numbered)
def showDifferences(differences, jirakeyprefix, outfh):
(revertslog, mergeslog, otherlog, numberedlog) = differences
printDivider(outfh)
printTabbed(outfh, jirakeyprefix + "-Numbers Log:")
for (gisnumber, descriptions) in sorted(numberedlog.items()):
printTabbed(outfh, gisnumber, 1)
descriptions = numberedlog[gisnumber];
for description in descriptions:
printTabbed(outfh, description, 2)
printDivider(outfh)
printTabbed(outfh, "Revert log:")
for reversion in sorted(revertslog):
printTabbed(outfh, reversion, 1)
printDivider(outfh)
printTabbed(outfh, "Merges log:")
for merge in sorted(mergeslog):
printTabbed(outfh, merge, 1)
printDivider(outfh)
printTabbed(outfh, "Other log:")
for other in sorted(otherlog):
printTabbed(outfh, other, 1)
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = '''%s
Created by dan on %s.
Copyright 2017 Global Inkjet Systems Ltd. All rights reserved.
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
try:
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]")
parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]")
parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]")
parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]")
parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]")
parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]")
# Process arguments
args = parser.parse_args()
try:
gitrootfolder = args.gitrootfolder
sourceversion = args.sourceversion
previousversion = args.previousversion
jirakeyprefix = args.jirakeyprefix
outputoptions = args.outputoptions
difffile = args.difffile
print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion)
print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile)
if difffile:
os.makedirs(os.path.dirname(difffile), exist_ok=True)
outfh = open(difffile, 'w')
else:
outfh = sys.stdout
printDivider(outfh, times = 4)
printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion))
gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion)
showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh)
if 'opposite' in outputoptions.lower():
printDivider(outfh, times = 4)
printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion))
gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion)
showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh)
if 'notcommon' in outputoptions.lower():
printDivider(outfh, times = 4)
printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion))
gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True)
showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh)
if difffile:
outfh.close()
return 0
except Exception as e:
print("Error while determining differences:", e.__repr__())
return 1
except Exception as e:
indent = len(program_name) * " "
print(program_name + ": " + repr(e) + "\n")
print(indent + " for help use --help")
return 2
if __name__ == "__main__":
sys.exit(main())

View file

@ -1,36 +1,122 @@
gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy")
versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy")
def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) {
copyArtifacts(
filter: filter,
excludes: excludes,
fingerprintArtifacts: fingerprint,
projectName: jenkinsjob.trim(),
selector: specific(buildnumber.toString()),
target: destpath
)
def importedmeta = meta.importrelated(destpath)
if (incrementversionnumber) {
meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))
class ImportOptions {
def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo)
boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported
String m_filter // Only import files/paths/folders that match the filter
String m_excludes // Exclude files/paths/folders that match the exclude filter
boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts
boolean m_nested // Import all child artifacts too (identified in the related meta info
boolean m_appendtoparentmeta // Append to parent meta
String m_sourceselect // Where are the artifacts coming from
String m_jobname
def m_buildnumberorselector
String m_repo
String m_revision
ImportOptions(script,
String sourceselect = 'Jenkins job artifact',
boolean incrementversionnumber = true,
String filter = '',
String excludes = '',
boolean fingerprint = false,
boolean nested = false,
appendtoparentmeta = true) {
m_script = script
m_sourceselect = sourceselect.trim()
m_incrementversionnumber = incrementversionnumber
m_filter = filter.trim()
m_excludes = excludes.trim()
m_fingerprint = fingerprint
m_nested = nested
m_appendtoparentmeta = appendtoparentmeta
}
void setsourceselect(String sourceselect) {
sourceselect = sourceselect.trim()
switch (sourceselect) {
case 'Jenkins job artefact':
m_sourceselect = 'Jenkins job artifact'
break
case 'Jenkins job artifact':
case 'Release repository':
m_sourceselect = sourceselect
break
default:
error("Unknown type of artifact source select: ${m_sourceselect}")
}
}
void setincrementversionnumber(boolean incrementversionnumber) {
m_incrementversionnumber = incrementversionnumber
}
void setfilter(String filter) {
m_filter = filter.trim()
}
void setexcludes(String excludes) {
m_excludes = excludes.trim()
}
void setfingerprint(boolean fingerprint) {
m_fingerprint = fingerprint
}
void setnested(boolean nested) {
m_nested = nested
}
void setappendtoparentmeta(boolean appendtoparentmeta) {
m_appendtoparentmeta = appendtoparentmeta
}
void jenkinsjob(String jobname, String buildnumberorselector) {
m_jobname = jobname.trim()
m_buildnumberorselector = buildnumberorselector.trim()
setsourceselect('Jenkins job artifact')
}
void releaserepo(String repo, String revision) {
m_repo = repo.trim()
m_revision = revision.trim()
setsourceselect('Release repository')
}
void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) {
jenkinsjob(jobname, buildnumberorselector)
releaserepo(repo, revision)
setsourceselect(sourceselect)
}
return importedmeta
}
def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) {
ImportOptions getNewImportOptions() {
return new ImportOptions(this)
}
def artifactimportfrombuildjob(meta, importoptions, String destpath) {
if (importoptions.m_sourceselect != 'Jenkins job artifact') {
error("Import options settings have not been set for: Jenkins job artifact")
return
}
copyArtifacts(
filter: filter,
excludes: excludes,
fingerprintArtifacts: fingerprint,
projectName: jenkinsjob.trim(),
filter: importoptions.m_filter,
excludes: importoptions.m_excludes,
fingerprintArtifacts: importoptions.m_fingerprint,
projectName: importoptions.m_jobname,
selector: [
$class: 'ParameterizedBuildSelector',
parameterName: buildselector
parameterName: importoptions.m_buildnumberorselector
],
target: destpath
)
def importedmeta = meta.importrelated(destpath)
if (incrementversionnumber) {
def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath)
if (importoptions.m_incrementversionnumber) {
meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))
}
return importedmeta
@ -42,16 +128,20 @@ def rewritemetafileasreleased(meta, String filepath) {
meta.saveJsonSerialised(filepath)
}
publisher = load("${autobuilder.commonscripts()}/publish/do.groovy")
def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) {
def rooturl = repo.trim()
def rootrevision = revision.trim()
def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) {
if (importoptions.m_sourceselect != 'Release repository') {
error("Import options settings have not been set for: Release repository")
return
}
def rooturl = importoptions.m_repo
def rootrevision = importoptions.m_revision
echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}"
def ArrayList<String> sparseitemlist = gh.convertSparseItemsString(filter)
def ArrayList<String> sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter)
gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist)
def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath))
def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath))
relatedimportinfo = importedmeta.getassociated()
echo "Cloning associated repos"
@ -76,46 +166,120 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d
echo "Marking root meta as released and rewriting meta file: ${metafilepath}"
rewritemetafileasreleased(importedmeta, metafilepath)
if (incrementversionnumber) {
if (importoptions.m_incrementversionnumber) {
meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))
}
return importedmeta
}
def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) {
def rooturl = repo.trim()
def rootrevision = revision.trim()
def artifactimportfromrepo(meta, importoptions, String destpath) {
if (importoptions.m_sourceselect != 'Release repository') {
error("Import options settings have not been set for: Release repository")
return
}
def rooturl = importoptions.m_repo
def rootrevision = importoptions.m_revision
echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}"
def ArrayList<String> sparseitemlist = gh.convertSparseItemsString(filter)
def ArrayList<String> sparseitemlist = gh.convertSparseItemsString(importoptions.m_filter)
gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist)
def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath))
def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath))
gh.stripgithistory(destpath)
def metafilepath = "${destpath}/${meta.metafilepath()}"
echo "Marking meta as released and rewriting meta file: ${metafilepath}"
rewritemetafileasreleased(importedmeta, metafilepath)
if (incrementversionnumber) {
if (importoptions.m_incrementversionnumber) {
meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))
}
return importedmeta
}
def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) {
switch (source) {
case "Jenkins job artifact":
return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint)
case "Release repository":
if (nested) {
return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint)
def artifactimport(meta, importoptions, String destpath) {
switch (importoptions.m_sourceselect) {
case 'Jenkins job artifact':
case 'Jenkins job artefact':
return artifactimportfrombuildjob(meta, importoptions, destpath)
case 'Release repository':
if (importoptions.m_nested) {
return hierarchyartifactimportfromrepo(meta, importoptions, destpath)
}
return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint)
return artifactimportfromrepo(meta, importoptions, destpath)
default:
error("Unknown type of artifact source: ${source}")
}
return meta
}
// Deprecated (use above)
def artifactimport(meta,
String sourceselect,
String jenkinsjob,
buildnumberorselector,
String repo,
String revision,
String destpath,
boolean incrementversionnumber = true,
String filter = '',
String excludes = '',
boolean fingerprint = false,
boolean nested = false) {
echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)"
def importoptions = getNewImportOptions()
importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision)
importoptions.setincrementversionnumber(incrementversionnumber)
importoptions.setfilter(filter)
importoptions.setexcludes(excludes)
importoptions.setfingerprint(fingerprint)
importoptions.setnested(nested)
return artifactimport(meta, importoptions, destpath)
}
// Deprecated (use above)
def artifactimport(meta,
String buildjob,
int buildnumber,
String destpath,
boolean incrementversionnumber = true,
String filter = '**',
String excludes = '',
boolean fingerprint = false) {
echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)"
def importoptions = getNewImportOptions()
importoptions.jenkinsjob(buildjob, "<SpecificBuildSelector plugin='copyartifact@1.41'> <buildNumber>${buildnumber}</buildNumber></SpecificBuildSelector>")
importoptions.setincrementversionnumber(incrementversionnumber)
importoptions.setfilter(filter)
importoptions.setexcludes(excludes)
importoptions.setfingerprint(fingerprint)
return artifactimport(meta, importoptions, destpath)
}
// Deprecated (use above)
def artifactimport(meta,
String buildjob,
buildnumberorselector,
String destpath,
boolean incrementversionnumber = true,
String filter = '',
String excludes = '',
boolean fingerprint = false) {
echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)"
def importoptions = getNewImportOptions()
importoptions.jenkinsjob(buildjob, buildnumberorselector)
importoptions.setincrementversionnumber(incrementversionnumber)
importoptions.setfilter(filter)
importoptions.setexcludes(excludes)
importoptions.setfingerprint(fingerprint)
return artifactimport(meta, importoptions, destpath)
}
return this;

View file

@ -23,10 +23,10 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin
}
// Given details about artifacts to import and details of installers to build, import artifacts that are required
def Map artifactsImporter(meta, artifactDetails, installerDetails) {
def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) {
artifactFetched = [:]
def importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
def importoptions = importer.getNewImportOptions()
for (installer in toBuild) {
def artifactKeys = installerDetails.get(installer)[1]
if (artifactKeys instanceof java.lang.String) {
@ -35,7 +35,12 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) {
for (artifactKey in artifactKeys) {
(sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey)
if (artifactFetched.get(artifactKey) == null) {
def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy)
importoptions.jenkinsjob(sourceProject, sourceselector)
importoptions.setincrementversionnumber(incrementversionnumber)
importoptions.setfilter(toCopy)
importoptions.setexcludes(notToCopy)
def importmeta = importer.artifactimport(meta, importoptions, destFolder)
artifactFetched.put(artifactKey, importmeta)
}
}

View file

@ -22,16 +22,14 @@ abstract class GISJsonSerialised implements Serializable {
class ProductInfo extends GISJsonSerialised {
String productname
String subcomponentname
String jirakey
ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") {
ProductInfo(script, productname = "", subcomponentname = "") {
super(script)
this.productname = productname
this.subcomponentname = subcomponentname
this.jirakey = jirakey
}
void set(String productname, String subcomponentname, String jirakey) {
void set(String productname, String subcomponentname) {
// Make sure the fields can be used in git tags
def int pstatus = 0
if (script.isUnix()) {
@ -55,20 +53,17 @@ class ProductInfo extends GISJsonSerialised {
}
this.productname = productname
this.subcomponentname = subcomponentname
this.jirakey = jirakey
}
void deserialiseJson(rawmeta) {
productname = rawmeta['productname']
subcomponentname = rawmeta['subcomponentname']
jirakey = rawmeta['jirakey']
}
LinkedHashMap serialisationdata() {
def data = [
'productname' : productname,
'subcomponentname' : subcomponentname,
'jirakey' : jirakey,
]
return data
}
@ -162,7 +157,7 @@ class BuildStampInfo extends GISJsonSerialised {
int revision
String buildstamp
BuildStampInfo(script, buildstampprefix = "1.30000.0.", revision = 0, buildstamp = "1.30000.0.0") {
BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") {
super(script)
this.buildstampprefix = buildstampprefix
this.revision = revision
@ -605,7 +600,7 @@ class MetaInfo extends GISJsonSerialised {
return METAFOLDERNAME + "/" + METAFILENAME
}
MetaInfo importrelated(String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') {
MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') {
MetaInfo mi = new MetaInfo(script)
def String adjustedpath = relativize(destpath)
@ -618,7 +613,9 @@ class MetaInfo extends GISJsonSerialised {
}
RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi)
associatedmeta.add(rmi)
if (appendtoparentmeta) {
associatedmeta.add(rmi)
}
return mi
}

View file

@ -4,6 +4,8 @@ unstablereason = ""
summaryfolder = "${WORKSPACE}/summary"
summaryfile = "${summaryfolder}/summary.txt"
summarystring = ""
repoandtagsfile = "${summaryfolder}/repoandtags.txt"
repoandtagsstring = ""
releaserepofolder = "${WORKSPACE}/releaserepos"
sourcerepofolder = "${WORKSPACE}/sourcerepos"
tempfolder = "${WORKSPACE}/temp"
@ -28,6 +30,8 @@ def preconditions() {
unstablereason = ""
summarystring = ""
writeFile(file: summaryfile, text: "Release summary: \n")
repoandtagsstring = ""
writeFile(file: repoandtagsfile, text: "Repository summary: \n")
}
def setunstable(String reason) {
@ -42,6 +46,12 @@ def summaryappend(String summary) {
summarystring += summary
}
def repoandtagsappend(String summary) {
def String repoandtagscontents = readFile(file: repoandtagsfile)
writeFile(file: repoandtagsfile, text: repoandtagscontents + summary)
repoandtagsstring += summary
}
def getpreviousreleasemeta(buildmeta, String checkoutfolder) {
def previousreleasemeta = buildmeta.getNewMetaObject(this)
// See if there is meta data file from the current release
@ -55,6 +65,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) {
def checkoutreleaserepo(buildmeta, String checkoutfolder) {
def String releaserepo = buildmeta.releaseinfo.releaserepoURL
// Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta
repoandtagsappend(" Release repo: ${releaserepo}")
def previousreleasemeta = buildmeta.getNewMetaObject(this)
dir(releaserepofolder) {
sh("git clone ${releaserepo} ${checkoutfolder}")
@ -70,6 +81,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) {
def checkoutsourcerepo(buildmeta, String checkoutfolder) {
def String sourcerepo = buildmeta.gitinfo.remoteurl
if (sourcerepo) {
repoandtagsappend(" Source repo: ${sourcerepo}")
// Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component
if (fileExists(checkoutfolder)) {
dir(checkoutfolder) {
@ -89,6 +101,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) {
sh("git clone ${sourcerepo} ${checkoutfolder}")
}
}
} else {
repoandtagsappend(" Source repo: None")
}
}
@ -365,6 +379,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri
def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}"
summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ")
repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ")
def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo)
checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo)
@ -374,6 +389,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri
}
summaryappend("\n")
repoandtagsappend("\n")
}
def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) {

View file

@ -66,7 +66,7 @@ pipeline {
echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml"
if (isUnix()) {
sh """#!/bin/bash
find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \;
find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \\;
# Uses mono
"${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml
"""

View file

@ -80,7 +80,8 @@ def String checkouttoreporoot(String gitURL) {
submoduleCfg: [],
])
} else {
echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status"
echo "Found ${gitURL} checked out at ${reporootsubfolder}"
/*
dir(reporootsubfolder) {
if (isUnix()) {
sh """
@ -94,6 +95,7 @@ def String checkouttoreporoot(String gitURL) {
"""
}
}
*/
}
}
return reporootsubfolder

View file

@ -67,14 +67,14 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) {
echo "Determining version number prefix from source branch name: ${sourcebranch}"
sourcebranch = sourcebranch.toLowerCase()
def defaultversionprefix = "1.30000.0."
def defaultversionprefix = "0.30000.0."
def splits = sourcebranch.split('/')
if (splits.size() < 1) {
echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}"
return defaultversionprefix
}
def mastertypeversionprefix = "1.20000.0."
def mastertypeversionprefix = "0.20000.0."
sourcebranch = splits.last()
if (sourcebranch.toLowerCase().find("master") != null) {
echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}"
@ -83,7 +83,7 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) {
def splits2 = sourcebranch.split('\\.')
if (splits2.size() < 1) {
def thirdpartyversionprefix = "1.25000.0."
def thirdpartyversionprefix = "0.25000.0."
echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}"
return thirdpartyversionprefix
}

View file

@ -8,7 +8,7 @@ def main(argv):
# Call once arguments to allow or disallow something
parser.add_argument('-f', '--filetoconvert', help='The file to convert')
parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace')
parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace')
parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern')
args = parser.parse_args(argv)