Merge commit '5bb08476af' into 1.10003.j20-comscr

# Conflicts:
#	autobuild/Common_scripts/versionnumbering/metaversion.groovy
This commit is contained in:
Damian Smith 2018-11-29 15:13:15 +00:00
commit 2f5617caea
8 changed files with 122 additions and 146 deletions

View file

@ -4,6 +4,7 @@ def combineSubjobArtifacts(meta, subJobs) {
def importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
def ArrayList<String> artifactFolders = new ArrayList<String>()
int revisiontotal = 0
for (subJob in subJobs) {
def subJobObject = subJob[0]
if (subJobObject != null) {
@ -13,11 +14,14 @@ def combineSubjobArtifacts(meta, subJobs) {
dir(destFolder) {
deleteDir()
}
importer.artifactimport(meta, subJobName, subJobNumber, destFolder)
importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder)
revisiontotal += importedmeta.buildstampinfo.revision
artifactFolders.add("${destFolder}/**")
}
}
return artifactFolders;
def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy")
revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1)
return [artifactFolders, revisiontotal]
}
return this;

View file

@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild"
commonscriptsroot = "${autobuildroot}/Common_scripts"
metafolderscript = null
def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") {
if (doclean) {
echo "Cleaning workspace"
@ -25,17 +27,21 @@ def getautobuildpath() {
return autobuildroot
}
def getmetafolderscript() {
return metafolderscript
}
// Start up boilderplate
def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) {
echo "Starting...."
autobuildroot = autobuildpath
commonscriptsroot = commonscriptspath
metafolderscript = load("${commonscripts()}/meta/folder.groovy")
conditionalclean()
def metafolder = load("${commonscripts()}/meta/folder.groovy")
def meta = metafolder.metareset(script)
def meta = getmetafolderscript().metareset(script)
meta.productinfo.set(productname, subcomponentname, jirakey)
def now = new Date()
now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC'))

View file

@ -1,16 +1,16 @@
def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) {
def String cs = autobuilder.commonscripts()
def String difffolder = meta.folder()
def String difffile = meta.difffilepath()
def String jirakey = meta.productinfo.jirakey
if (isUnix()) {
sh """
"${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}"
"${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}"
"""
}
else {
bat """
"${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}"
"${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}"
"""
}
}

View file

@ -29,10 +29,6 @@ class GitException(Exception):
def __repr__(self):
return self.message
def makeFolder(difffolder):
print("Creating folder (if not already there): {0}".format(difffolder))
os.makedirs(difffolder, exist_ok = True)
def printDivider(outfh, times = 1, length = 100):
for _ in range(times):
outfh.write("{0}\n".format("*" * length))
@ -84,6 +80,9 @@ jiraprojectpattern = {
'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'),
),
'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"),
re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'),
),
}
mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$")
@ -172,7 +171,7 @@ USAGE
parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]")
parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]")
parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]")
parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]")
parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]")
# Process arguments
args = parser.parse_args()
@ -183,14 +182,13 @@ USAGE
previousversion = args.previousversion
jirakeyprefix = args.jirakeyprefix
outputoptions = args.outputoptions
difffolder = args.difffolder
difffile = args.difffile
print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion)
print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder)
print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile)
if difffolder:
makeFolder(difffolder)
outfh = open(os.path.join(difffolder, "differences.txt"), 'w')
if difffile:
outfh = open(difffile, 'w')
else:
outfh = sys.stdout
@ -209,7 +207,7 @@ USAGE
gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True)
showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh)
if difffolder:
if difffile:
outfh.close()
return 0
except Exception as e:

View file

@ -7,7 +7,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St
selector: specific(buildnumber.toString()),
target: destpath
)
meta.importrelated(destpath)
return meta.importrelated(destpath)
}
def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) {
@ -21,7 +21,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat
],
target: destpath
)
meta.importrelated(destpath)
return meta.importrelated(destpath)
}
return this;

View file

@ -19,8 +19,8 @@ abstract class GISJsonSerialised implements Serializable {
}
class ProductInfo extends GISJsonSerialised {
private String productname
private String subcomponentname
String productname
String subcomponentname
String jirakey
ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") {
@ -137,30 +137,40 @@ class JenkinsBuildInfo extends GISJsonSerialised {
class BuildStampInfo extends GISJsonSerialised {
private String buildstampprefix
private int revision
int revision
String buildstamp
String buildstampcommasep
BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") {
BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") {
super(script)
this.buildstampprefix = buildstampprefix
this.revision = revision
this.buildstamp = buildstamp
this.buildstampcommasep = buildstampcommasep
}
void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) {
void set(String buildstampprefix, int revision, String buildstamp) {
this.buildstampprefix = buildstampprefix
this.revision = revision
this.buildstamp = buildstamp
this.buildstampcommasep = buildstampcommasep
}
String buildstampcommasep() {
def String buildstampcommasep = buildstamp.replace(".", ",")
return buildstampcommasep
}
void changerevision(int newrevision) {
script.echo("Build stamp currently ${buildstamp}")
revision = newrevision
def List buildstampsplit = buildstamp.split('\\.')
buildstampsplit[-1] = newrevision.toString()
buildstamp = buildstampsplit.join('.')
script.echo("Build stamp now ${buildstamp}")
}
void deserialiseJson(rawmeta) {
buildstampprefix = rawmeta['buildstampprefix']
revision = rawmeta['revision']
buildstamp = rawmeta['buildstamp']
buildstampcommasep = rawmeta['buildstampcommasep']
}
LinkedHashMap serialisationdata() {
@ -168,15 +178,14 @@ class BuildStampInfo extends GISJsonSerialised {
'buildstampprefix' : buildstampprefix,
'revision' : revision,
'buildstamp' : buildstamp,
'buildstampcommasep' : buildstampcommasep,
]
return data
}
}
class ReleaseInfo extends GISJsonSerialised {
private String releaserepoURL
private ArrayList<String> releaseitems
String releaserepoURL
ArrayList<String> releaseitems
ReleaseInfo(script, releaserepoURL = "") {
super(script)
@ -376,10 +385,11 @@ class PublishableInfo extends GISJsonSerialised {
}
class MetaInfo extends GISJsonSerialised {
private String FORMAT = "0.1"
private String METAFOLDERNAME = "meta"
private String METAFILENAME = "info.json"
private String METAARTIFACTS = "${METAFOLDERNAME}/**"
private static String FORMAT = "0.1"
private static String METAFOLDERNAME = "meta"
private static String METAFILENAME = "info.json"
private static String METADIFFFILENAME = "differences.txt"
private static String METAARTIFACTS = "${METAFOLDERNAME}/**"
class RelatedMetaInfo extends GISJsonSerialised {
String offsetpath
@ -507,10 +517,14 @@ class MetaInfo extends GISJsonSerialised {
return path
}
String metafilepath() {
static String metafilepath() {
return METAFOLDERNAME + "/" + METAFILENAME
}
static String difffilepath() {
return METAFOLDERNAME + "/" + METADIFFFILENAME
}
MetaInfo importrelated(String destpath) {
MetaInfo mi = new MetaInfo(script)
@ -523,6 +537,17 @@ class MetaInfo extends GISJsonSerialised {
RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi)
associatedmeta.add(rmi)
def importedmetadifffile = adjustedpath + "/" + difffilepath()
if (script.fileExists(importedmetadifffile)) {
def String newcontents = script.readFile(file: importedmetadifffile)
def String currentcontents = ""
if (script.fileExists(difffilepath())) {
currentcontents = script.readFile(file: difffilepath())
}
script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents)
}
return mi
}
@ -563,6 +588,42 @@ class MetaInfo extends GISJsonSerialised {
saveJsonSerialised(metafilepath)
return artifactlist
}
// Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java
private ArrayList<RelatedMetaInfo> releasewalkerp(int maxdepth, ArrayList<RelatedMetaInfo> releasemetacollection, String pathsofar) {
if (maxdepth > 0) {
if (releaseinfo.releaserepoURL) {
RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this)
releasemetacollection.add(rmi)
}
if (pathsofar) {
pathsofar += "/"
}
for (relatedmetainfo in associatedmeta) {
MetaInfo othermeta = relatedmetainfo.meta
String newpathsofar = pathsofar + relatedmetainfo.offsetpath
releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar)
}
}
return releasemetacollection
}
ArrayList<RelatedMetaInfo> releasewalker(int maxdepth, String startpath = "") {
ArrayList<RelatedMetaInfo> releasemetacollection = new ArrayList<RelatedMetaInfo>()
releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath)
return releasemetacollection
}
Map<String, RelatedMetaInfo> releasewalkerunique(int maxdepth, String startpath = "") {
Map<String, RelatedMetaInfo> releasemetauniquecollection = new LinkedHashMap<String, RelatedMetaInfo>()
ArrayList<RelatedMetaInfo> releasemetacollection = new ArrayList<RelatedMetaInfo>()
releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath)
for (relatedmetainfo in releasemetacollection) {
String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname
releasemetauniquecollection.put(uniquestring, relatedmetainfo)
}
return releasemetauniquecollection
}
}
MetaInfo metareset(script) {
@ -575,4 +636,10 @@ MetaInfo metareset(script) {
return mi
}
MetaInfo metafromfolder(String folder = "") {
MetaInfo buildmeta = new MetaInfo(this)
buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}")
return buildmeta
}
return this;

View file

@ -1,15 +1,18 @@
def int maxversionnumber() {
def int MAXVERSIONNUMBER = 65535
return MAXVERSIONNUMBER
}
def getGitRevision(String gitrootpath, int basebuildnumber) {
def int adjustedchangecount
def String gitrevision
def String remoteinfo
def String remoteurl
dir( gitrootpath) {
def int MAXVERSIONNUMBER = 65535
def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim()
echo "Change count: ${changecount}"
adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1)
adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1)
echo "Adjusted change count: ${adjustedchangecount}"
gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
@ -18,7 +21,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) {
remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim()
echo "Remote info: ${remoteinfo}"
remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim()
remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim()
echo "Remote URL: ${remoteurl}"
}
return [adjustedchangecount, gitrevision, remoteinfo, remoteurl]
@ -28,8 +31,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) {
// origin/master
// origin/2.0.x
def getVersionPrefix(String sourcebranch) {
def int MAXVERSIONNUMBER = 65535
echo "Determining version number prefix from source branch name: ${sourcebranch}"
sourcebranch = sourcebranch.toLowerCase()
@ -65,7 +66,7 @@ def getVersionPrefix(String sourcebranch) {
def integers = []
for (integerstring in integerstrings) {
int integer = integerstring.toInteger()
integer = integer % (MAXVERSIONNUMBER + 1)
integer = integer % (maxversionnumber() + 1)
integers.add(integer)
}
@ -112,7 +113,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum
echo "Build stamp is: ${buildstamp}"
meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo)
meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ","))
meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp)
return buildstamp
}

View file

@ -1,100 +0,0 @@
def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy")
def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy")
def prefixline(rootpath, filepath) {
def fileContents = readFile("${filepath}").trim()
def fileContentsNormalized = fileContents.normalize()
def newFileContents = ""
def lines = fileContentsNormalized.readLines()
for (aline in lines) {
newFileContents += "${rootpath}" + "/" + aline + "${nl}"
}
return newFileContents
}
def mergemeta(rootpath, versionFolder, outputFolder) {
echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}"
def differencesFile = "${versionFolder}/differences.txt"
def newContents = readFile file: "${differencesFile}"
def metadifferencesFile = "${outputFolder}/differences.txt"
writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents
def buildStampFile = "${versionFolder}/buildstamps.txt"
def buildStampFileContents = prefixline(rootpath, "${buildStampFile}")
def metabuildStampFile = "${outputFolder}/buildstamps.txt"
writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents
def intrevision = 0
if (fileExists("${versionFolder}/revision.txt")) {
def revision = readFile("${versionFolder}/revision.txt").trim()
intrevision = revision.toInteger()
echo "Revision read in: ${intrevision}"
}
else {
intrevision = 1
}
return intrevision
}
def appendcomponentinfo(rootpath, versionFolder, outputFolder) {
echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}"
def differencesFile = "${versionFolder}/differences.txt"
if (fileExists("${differencesFile}")) {
def newContents = readFile file: "${differencesFile}"
def metadifferencesFile = "${outputFolder}/differences.txt"
writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents
}
def buildStampFile = "${versionFolder}/buildstamps.txt"
def buildStampFileContents = readFile("${buildStampFile}").trim()
def remoteurlFile = "${versionFolder}/remoteurl.txt"
def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim()
def gitRevisionFile = "${versionFolder}/gitrevision.txt"
def gitRevisionFileContents = readFile("${gitRevisionFile}").trim()
def releaseJobNameFile = "${versionFolder}/releasejobname.txt"
def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim()
def metabuildStampFile = "${outputFolder}/buildstamps.txt"
writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents
def revision = readFile("${versionFolder}/revision.txt").trim()
def intrevision = revision.toInteger()
echo "Revision read in: ${intrevision}"
return intrevision
}
// Each version folder should have been created using version creation scripts in this Common_scripts folder
// as their are well-known files in each folder
def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") {
def totalrevision = 0
dir("${outputFolder}") {
deleteDir()
}
for (artifactItem in artifactList) {
(rootpath, versionFolder) = artifactItem
echo "Inspecting folder: ${versionFolder}"
def metaversionsignaturefile = "${versionFolder}/metaversion.txt"
// If the sub project is a meta job
def intrevision = 0
if (fileExists("${metaversionsignaturefile}")) {
intrevision = mergemeta(rootpath, versionFolder, outputFolder)
}
else {
intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder)
}
totalrevision += intrevision
}
echo "Accumulated revision is: ${totalrevision}"
writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString()
writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString())
}
return this;