From 8d73541b51b48be36b26edc337bbf7b52538d76f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Oct 2018 14:53:02 +0100 Subject: [PATCH 001/195] Using SM200 master (714fd686bf5ea07f83346342362eca45c6ffc9df) as a starting point for HMB-FD-HV autobuild scripts --- .project | 17 ++ .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + commandline/commandline.gsh | 13 ++ .../eclipsebuild/disableparallelbuilds.gsh | 12 + compile/eclipsebuild/eclipseds5build.gsh | 180 ++++++++++++++ configuration/configuration.gsh | 37 +++ differences/.project | 17 ++ differences/.pydevproject | 8 + differences/show_release_differences.py | 220 ++++++++++++++++++ test.gsh | 7 + versioncontrol/git.gsh | 63 +++++ versionnumbering/versioninfo.py | 219 +++++++++++++++++ versionstamping/.project | 17 ++ versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 41 ++++ 16 files changed, 869 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 commandline/commandline.gsh create mode 100644 compile/eclipsebuild/disableparallelbuilds.gsh create mode 100644 compile/eclipsebuild/eclipseds5build.gsh create mode 100644 configuration/configuration.gsh create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/show_release_differences.py create mode 100644 test.gsh create mode 100644 versioncontrol/git.gsh create mode 100644 versionnumbering/versioninfo.py create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/commandline/commandline.gsh b/commandline/commandline.gsh new file mode 100644 index 0000000..df99a75 --- /dev/null +++ b/commandline/commandline.gsh @@ -0,0 +1,13 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return bat(returnStdout: true, script: """ + @echo off + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py new file mode 100644 index 0000000..93d540b --- /dev/null +++ b/differences/show_release_differences.py @@ -0,0 +1,220 @@ +''' +showreleasedifferences -- shortdesc + +showreleasedifferences generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import subprocess +import os +import sys +import re + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") + +class GitException(Exception): + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + +def makeFolder(versionfolder): + print("Creating folder (if not already there): {0}".format(versionfolder)) + os.makedirs(versionfolder, exist_ok = True) + +def printDivider(outfh, times = 1, length = 100): + for _ in range(times): + outfh.write("{0}\n".format("*" * length)) + +TABSIZE = 4 +def printTabbed(outfh, string, tabs = 0): + outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) + +def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): + if tripledot: + releaseExpression = previousRelease + "..." + currentRelease + else: + releaseExpression = previousRelease + ".." + currentRelease + currentdir = os.getcwd() + os.chdir(gitrootfolder) + gitlog = [] + printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if line: + linestring = line.decode('utf-8').rstrip() + gitlog.append(linestring) + #printTabbed(outfh, linestring, 1) + else: + break + proc.wait() + if proc.returncode > 0: + raise GitException("Git executable returned an error") + os.chdir(currentdir) + return gitlog + +jiraprojectpattern = { + 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), + ), + 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), + ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), + } + +mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") + +def processLog(gitlog, jirakeyprefix): + reverts = [] + merges = {} + other = [] + numbered = {} + (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] + for line in gitlog: + expectedmatch = expectedpattern.match(line) + revertmatch = revertpattern.match(line) + mergematch = mergepattern.match(line) + if expectedmatch == None and revertmatch == None and mergematch == None: + other.append(line) + if revertmatch: + gisnumber = jirakeyprefix + "-" + revertmatch.group(2) + description = revertmatch.group(3) + reverts.append("Revert: {0} {1}".format(gisnumber, description)) + if mergematch: + merge = mergematch.group(2) + if not merge in merges: + merges[merge] = 1 + else: + merges[merge] += 1 + if expectedmatch: + gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) + description = expectedmatch.group(2) + if not gisnumber in numbered: + numbered[gisnumber] = [] + numbered[gisnumber].append(description) + return (reverts, merges, other, numbered) + +def showDifferences(differences, jirakeyprefix, outfh): + (revertslog, mergeslog, otherlog, numberedlog) = differences + printDivider(outfh) + printTabbed(outfh, jirakeyprefix + "-Numbers Log:") + for (gisnumber, descriptions) in sorted(numberedlog.items()): + printTabbed(outfh, gisnumber, 1) + descriptions = numberedlog[gisnumber]; + for description in descriptions: + printTabbed(outfh, description, 2) + printDivider(outfh) + printTabbed(outfh, "Revert log:") + for reversion in sorted(revertslog): + printTabbed(outfh, reversion, 1) + printDivider(outfh) + printTabbed(outfh, "Merges log:") + for merge in sorted(mergeslog): + printTabbed(outfh, merge, 1) + printDivider(outfh) + printTabbed(outfh, "Other log:") + for other in sorted(otherlog): + printTabbed(outfh, other, 1) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") + parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") + parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") + parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + gitrootfolder = args.gitrootfolder + sourceversion = args.sourceversion + previousversion = args.previousversion + jirakeyprefix = args.jirakeyprefix + outputoptions = args.outputoptions + versionfolder = args.versionfolder + + print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + + if versionfolder: + makeFolder(versionfolder) + outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + else: + outfh = sys.stdout + + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'opposite' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'notcommon' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + + if versionfolder: + outfh.close() + return 0 + except Exception as e: + print("Error while determining differences:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..23fcfc0 --- /dev/null +++ b/test.gsh @@ -0,0 +1,7 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; + diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh new file mode 100644 index 0000000..dbc09ff --- /dev/null +++ b/versioncontrol/git.gsh @@ -0,0 +1,63 @@ +clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git log -1 --format=oneline') + } +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + ], + submoduleCfg: [], + ]) +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() <= 1) { + return false + } else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +return this diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..a9d8738 --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,219 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision, remoteinfo) + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return mastertypeversionprefix + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..3e59c6f --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,41 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + filecopy = filetoconvert + ".orig" + try: + shutil.move(filetoconvert, filecopy) + except FileNotFoundError: + print("Could not find file: {0}\n".format(filetoconvert)) + return 1 + + with open(filetoconvert, 'x') as f: + with open(filecopy,'r') as c: + fileread = c.read() + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + #print ("DEBUG: %s "%converted) + print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + f.write(converted) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 0512a83425de4e8a056b14fb8a5b2dc610201328 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 11 Oct 2018 20:11:30 +0100 Subject: [PATCH 002/195] HMBFDHV-12 - Add HMBFDHV differences search --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 93d540b..4937134 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -78,6 +78,9 @@ jiraprojectpattern = { 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), ), + 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From 35cd5c63c6a3dc3074552324c4c387a36d5534f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 15 Oct 2018 15:54:16 +0100 Subject: [PATCH 003/195] Merge commit 'fadf5e954325f20d50142038be5aa428ad5d31af' into production --- .project | 17 ++ .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + commandline/commandline.gsh | 13 ++ .../eclipsebuild/disableparallelbuilds.gsh | 12 + compile/eclipsebuild/eclipseds5build.gsh | 186 +++++++++++++++ configuration/configuration.gsh | 37 +++ differences/.project | 17 ++ differences/.pydevproject | 8 + differences/show_release_differences.py | 220 ++++++++++++++++++ test.gsh | 7 + versioncontrol/git.gsh | 63 +++++ versionnumbering/versioninfo.py | 219 +++++++++++++++++ versionstamping/.project | 17 ++ versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 41 ++++ 16 files changed, 875 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 commandline/commandline.gsh create mode 100644 compile/eclipsebuild/disableparallelbuilds.gsh create mode 100644 compile/eclipsebuild/eclipseds5build.gsh create mode 100644 configuration/configuration.gsh create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/show_release_differences.py create mode 100644 test.gsh create mode 100644 versioncontrol/git.gsh create mode 100644 versionnumbering/versioninfo.py create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/commandline/commandline.gsh b/commandline/commandline.gsh new file mode 100644 index 0000000..df99a75 --- /dev/null +++ b/commandline/commandline.gsh @@ -0,0 +1,13 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return bat(returnStdout: true, script: """ + @echo off + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py new file mode 100644 index 0000000..93d540b --- /dev/null +++ b/differences/show_release_differences.py @@ -0,0 +1,220 @@ +''' +showreleasedifferences -- shortdesc + +showreleasedifferences generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import subprocess +import os +import sys +import re + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") + +class GitException(Exception): + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + +def makeFolder(versionfolder): + print("Creating folder (if not already there): {0}".format(versionfolder)) + os.makedirs(versionfolder, exist_ok = True) + +def printDivider(outfh, times = 1, length = 100): + for _ in range(times): + outfh.write("{0}\n".format("*" * length)) + +TABSIZE = 4 +def printTabbed(outfh, string, tabs = 0): + outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) + +def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): + if tripledot: + releaseExpression = previousRelease + "..." + currentRelease + else: + releaseExpression = previousRelease + ".." + currentRelease + currentdir = os.getcwd() + os.chdir(gitrootfolder) + gitlog = [] + printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if line: + linestring = line.decode('utf-8').rstrip() + gitlog.append(linestring) + #printTabbed(outfh, linestring, 1) + else: + break + proc.wait() + if proc.returncode > 0: + raise GitException("Git executable returned an error") + os.chdir(currentdir) + return gitlog + +jiraprojectpattern = { + 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), + ), + 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), + ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), + } + +mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") + +def processLog(gitlog, jirakeyprefix): + reverts = [] + merges = {} + other = [] + numbered = {} + (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] + for line in gitlog: + expectedmatch = expectedpattern.match(line) + revertmatch = revertpattern.match(line) + mergematch = mergepattern.match(line) + if expectedmatch == None and revertmatch == None and mergematch == None: + other.append(line) + if revertmatch: + gisnumber = jirakeyprefix + "-" + revertmatch.group(2) + description = revertmatch.group(3) + reverts.append("Revert: {0} {1}".format(gisnumber, description)) + if mergematch: + merge = mergematch.group(2) + if not merge in merges: + merges[merge] = 1 + else: + merges[merge] += 1 + if expectedmatch: + gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) + description = expectedmatch.group(2) + if not gisnumber in numbered: + numbered[gisnumber] = [] + numbered[gisnumber].append(description) + return (reverts, merges, other, numbered) + +def showDifferences(differences, jirakeyprefix, outfh): + (revertslog, mergeslog, otherlog, numberedlog) = differences + printDivider(outfh) + printTabbed(outfh, jirakeyprefix + "-Numbers Log:") + for (gisnumber, descriptions) in sorted(numberedlog.items()): + printTabbed(outfh, gisnumber, 1) + descriptions = numberedlog[gisnumber]; + for description in descriptions: + printTabbed(outfh, description, 2) + printDivider(outfh) + printTabbed(outfh, "Revert log:") + for reversion in sorted(revertslog): + printTabbed(outfh, reversion, 1) + printDivider(outfh) + printTabbed(outfh, "Merges log:") + for merge in sorted(mergeslog): + printTabbed(outfh, merge, 1) + printDivider(outfh) + printTabbed(outfh, "Other log:") + for other in sorted(otherlog): + printTabbed(outfh, other, 1) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") + parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") + parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") + parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + gitrootfolder = args.gitrootfolder + sourceversion = args.sourceversion + previousversion = args.previousversion + jirakeyprefix = args.jirakeyprefix + outputoptions = args.outputoptions + versionfolder = args.versionfolder + + print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + + if versionfolder: + makeFolder(versionfolder) + outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + else: + outfh = sys.stdout + + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'opposite' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'notcommon' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + + if versionfolder: + outfh.close() + return 0 + except Exception as e: + print("Error while determining differences:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..23fcfc0 --- /dev/null +++ b/test.gsh @@ -0,0 +1,7 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; + diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh new file mode 100644 index 0000000..dbc09ff --- /dev/null +++ b/versioncontrol/git.gsh @@ -0,0 +1,63 @@ +clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git log -1 --format=oneline') + } +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + ], + submoduleCfg: [], + ]) +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() <= 1) { + return false + } else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +return this diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..a9d8738 --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,219 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision, remoteinfo) + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return mastertypeversionprefix + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..3e59c6f --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,41 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + filecopy = filetoconvert + ".orig" + try: + shutil.move(filetoconvert, filecopy) + except FileNotFoundError: + print("Could not find file: {0}\n".format(filetoconvert)) + return 1 + + with open(filetoconvert, 'x') as f: + with open(filecopy,'r') as c: + fileread = c.read() + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + #print ("DEBUG: %s "%converted) + print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + f.write(converted) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 1d9684363a4cc0efe5dadc3b18269c11a1f76a3b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 16 Oct 2018 08:50:30 +0100 Subject: [PATCH 004/195] Merge commit 'fadf5e954325f20d50142038be5aa428ad5d31af' --- compile/eclipsebuild/eclipseds5build.gsh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9fd5293..3061fea 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -154,7 +154,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } - doExport(exportfolder, exportpathlist, "", build_variant) + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." + } } catch (Exception e) { echo "Build failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') @@ -166,7 +169,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - doExport(exportfolder, exportpathlist, project) + result = doExport(exportfolder, exportpathlist, project) + if (result != 0) { + error "Failed to copy...." + } } } } From d202c89bfa3fc0e2ce83bf65cd7e0872246c53eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 10:07:49 +0100 Subject: [PATCH 005/195] Move json file comparison to subtree area --- differences/json.gsh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 differences/json.gsh diff --git a/differences/json.gsh b/differences/json.gsh new file mode 100644 index 0000000..47d9cc5 --- /dev/null +++ b/differences/json.gsh @@ -0,0 +1,15 @@ +def comparefiles(importedjsonfile, currentjsonfile) { + env.IMPORTED_JSON="${importedjsonfile}" + env.CURRENT_JSON="${currentjsonfile}" + if (isUnix()) { + sh """ + # jq -S sorts json output + diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + """ + } + else { + error "Windows not currently supporting json file comparison" + } +} + +return this; From 2393df8cb20b15b5a60ed6a6989263189df86005 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 13:32:04 +0100 Subject: [PATCH 006/195] Move meta artifact builder to common scripts subtree area --- artifacts/metaexport.gsh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 artifacts/metaexport.gsh diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.gsh new file mode 100644 index 0000000..368871e --- /dev/null +++ b/artifacts/metaexport.gsh @@ -0,0 +1,24 @@ + +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKSPACE}") { + artifactFolders = "" + for (subJob in subJobs) { + subJobObject = subJob[0] + artifactFolder = subJob[1] + if (subJobObject != null) { + subJobName = subJobObject.getProjectName() + subJobNumber = subJobObject.getNumber() + destFolder = "${exportRootFolder}/${artifactFolder}" + + dir("${destFolder}") { + deleteDir() + } + + echo "Copying artifacts from: Job: ${subJobName} Number: ${subJobNumber} to folder: ${destFolder}" + copyArtifacts fingerprintArtifacts: true, projectName: "${subJobName}", selector: specific("${subJobNumber}"), target: "${destFolder}" + artifactFolders += "${destFolder}/**," + } + } +} + +return this; From 18c1343663e761a9733f739c2bb6de4b3c783eea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 14:24:17 +0100 Subject: [PATCH 007/195] Move meta artifact builder to common scripts subtree area --- artifacts/metaexport.gsh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.gsh index 368871e..66a8153 100644 --- a/artifacts/metaexport.gsh +++ b/artifacts/metaexport.gsh @@ -1,6 +1,6 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKSPACE}") { +def combineSubjobArtifacts(subJobs) { artifactFolders = "" for (subJob in subJobs) { subJobObject = subJob[0] @@ -8,7 +8,7 @@ def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKS if (subJobObject != null) { subJobName = subJobObject.getProjectName() subJobNumber = subJobObject.getNumber() - destFolder = "${exportRootFolder}/${artifactFolder}" + destFolder = "${artifactFolder}" dir("${destFolder}") { deleteDir() @@ -19,6 +19,7 @@ def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKS artifactFolders += "${destFolder}/**," } } + return artifactFolders; } return this; From e7c44d4fd43da49cf3468a18a511757e61f3f650 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Oct 2018 00:09:04 +0100 Subject: [PATCH 008/195] Add utility to make build stamp representing a set of components --- versionnumbering/complexversioninfo.py | 220 +++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 versionnumbering/complexversioninfo.py diff --git a/versionnumbering/complexversioninfo.py b/versionnumbering/complexversioninfo.py new file mode 100644 index 0000000..be6ada4 --- /dev/null +++ b/versionnumbering/complexversioninfo.py @@ -0,0 +1,220 @@ +# encoding: utf-8 +''' +complexversionnumbering -- shortdesc + +complexversionnumbering generates a version folder with a common set of version information files in it for a complex component (consisting of many subcomponents) + +@author: dan + +@copyright: 2018 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2018-10-24' +__updated__ = '2018-10-24' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision, remoteinfo) + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return mastertypeversionprefix + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-p", "--stampprefix", dest="stampprefix", action="store", default="1.30000.0", help="A prefix to help identify this build [default: %(default)s]") + parser.add_argument("-s", "--subbuildstamp", dest="subbuildstamp", nargs='+', action="store", help="A subcomponent build stamp - there must be 1 or more") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + stampprefix = args.stampprefix + subbuildstamp = args.subbuildstamp + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Stamp prefix:", stampprefix, "sub build stamp:", subbuildstamp, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) From 1d959d0a85015f21525ec5ebaf656d5c118c8e11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Oct 2018 23:04:31 +0100 Subject: [PATCH 009/195] Detect branches with 'master' string better; Split numbers better in branch names --- versionnumbering/versioninfo.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index a9d8738..7e6043f 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -12,6 +12,7 @@ import os import sys import shutil import subprocess +import re from argparse import ArgumentParser from argparse import RawDescriptionHelpFormatter @@ -81,13 +82,18 @@ def getVersionPrefix(sourcebranch): sourcebranch = splits[-1] - if sourcebranch.lower().endswith("master"): # Special case + if sourcebranch.lower().find("master") != -1: # Special case return mastertypeversionprefix - splits = sourcebranch.split('.') + splits2 = re.sourcebranch(r'(\d+)', v) + splits = [] + for split in splits2: + if split.isdigit(): + splits.append(split) + versionprefix = "" if len(splits) < 1: - print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) versionprefix = defaultversionprefix else: splitnumbercount = 0 From 9a4601e41e9fe627df818126f6c241a91bf01b5e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Oct 2018 23:12:00 +0100 Subject: [PATCH 010/195] Detect branches with 'master' string better; Split numbers better in branch names --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 7e6043f..b0d8ba1 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -85,7 +85,7 @@ def getVersionPrefix(sourcebranch): if sourcebranch.lower().find("master") != -1: # Special case return mastertypeversionprefix - splits2 = re.sourcebranch(r'(\d+)', v) + splits2 = re.split(r'(\d+)', sourcebranch) splits = [] for split in splits2: if split.isdigit(): From 883ea65599072260c7d13e61c1cf792115e51d83 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 00:19:26 +0100 Subject: [PATCH 011/195] Add metaversion generation --- versionnumbering/complexversioninfo.py | 220 ------------------------- versionnumbering/metaversion.gsh | 41 +++++ 2 files changed, 41 insertions(+), 220 deletions(-) delete mode 100644 versionnumbering/complexversioninfo.py create mode 100644 versionnumbering/metaversion.gsh diff --git a/versionnumbering/complexversioninfo.py b/versionnumbering/complexversioninfo.py deleted file mode 100644 index be6ada4..0000000 --- a/versionnumbering/complexversioninfo.py +++ /dev/null @@ -1,220 +0,0 @@ -# encoding: utf-8 -''' -complexversionnumbering -- shortdesc - -complexversionnumbering generates a version folder with a common set of version information files in it for a complex component (consisting of many subcomponents) - -@author: dan - -@copyright: 2018 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2018-10-24' -__updated__ = '2018-10-24' - -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -slaveMachine = os.getenv("NODE_NAME", default="-") - -def resetFolder(versionfolder): - print("Removing folders and contents of: {0}".format(versionfolder)) - try: - shutil.rmtree(versionfolder) - except FileNotFoundError: - pass - - print("Creating folder: {0}".format(versionfolder)) - os.makedirs(versionfolder) - -def getGitRevision(gitrootpath, basenumber): - cwd = os.getcwd() - - path = os.path.expanduser(gitrootpath) - print("Changing current working directory to: {0}".format(path)) - os.chdir(path) - - print("Determining revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - changecount = completed.stdout.decode("utf-8").strip() - print("Change count: {0}".format(changecount)) - adjustedchangecount = int(changecount) + int(basenumber) - print("Adjusted change count: {0}".format(adjustedchangecount)) - - print("Determining current revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - revision = completed.stdout.decode("utf-8").strip() - print("Revision: {0}".format(revision)) - - print("Determining remote info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteinfo = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteinfo)) - - os.chdir(cwd) - print("Restored current working directory to: {0}".format(cwd)) - - return (adjustedchangecount, revision, remoteinfo) - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().endswith("master"): # Special case - return mastertypeversionprefix - - splits = sourcebranch.split('.') - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-p", "--stampprefix", dest="stampprefix", action="store", default="1.30000.0", help="A prefix to help identify this build [default: %(default)s]") - parser.add_argument("-s", "--subbuildstamp", dest="subbuildstamp", nargs='+', action="store", help="A subcomponent build stamp - there must be 1 or more") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") - parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - stampprefix = args.stampprefix - subbuildstamp = args.subbuildstamp - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber - - print("Stamp prefix:", stampprefix, "sub build stamp:", subbuildstamp, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) - - resetFolder(versionfolder) - - (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh new file mode 100644 index 0000000..b4b1ad7 --- /dev/null +++ b/versionnumbering/metaversion.gsh @@ -0,0 +1,41 @@ + +// Each version folder should have been created using version creation scripts in this Common_scripts folder +// as their are well-known files in each folder +def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = "${WORKSPACE}/metaversion") { + env.VLP = versionLabelPrefix + env.OF = outputFolder + totalrevision = 0 + if (isUnix()) { + sh """#!/bin/bash + + rm -rf "${OF}" + mkdir -p "${OF}" + if [ \$? -ne 0 ]; then exit 1; fi + + touch "${OF}/differences.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + else { + error "combineSubjobVersions script is not supported in Windows yet" + } + for (versionFolder in versionFolderList) { + env.differemcesFile = "${versionFolder}/differences.txt" + sh """#!/bin/bash + + echo ${differencesFile} >> "${OF}/differences.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + cat "${differencesFile}" >> "${OF}/differences.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ + revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + totalrevision += intrevision + echo "Accumulated revision is: ${totalrevision}" + } + writeFile file: "${OF}/metaversion.txt" text: versionLabelPrefix + "." + totalrevision +} + +return this; \ No newline at end of file From 55ef090a86d3f317b7a85189dc6cfca2808a7870 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 08:37:10 +0100 Subject: [PATCH 012/195] Add metaversion generation --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index b4b1ad7..c6eee74 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -35,7 +35,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = totalrevision += intrevision echo "Accumulated revision is: ${totalrevision}" } - writeFile file: "${OF}/metaversion.txt" text: versionLabelPrefix + "." + totalrevision + writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } return this; \ No newline at end of file From bca6d804dd82173d1e1bacc54ff0dee8ed7f62ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 09:13:34 +0100 Subject: [PATCH 013/195] Add metaversion generation --- versionnumbering/metaversion.gsh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index c6eee74..e1cdaf6 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -14,28 +14,46 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = touch "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi + + touch "${OF}/buildstamps.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ } else { error "combineSubjobVersions script is not supported in Windows yet" } for (versionFolder in versionFolderList) { - env.differemcesFile = "${versionFolder}/differences.txt" + env.differencesFile = "${versionFolder}/differences.txt" sh """#!/bin/bash - echo ${differencesFile} >> "${OF}/differences.txt" + echo ${differencesFile}: >> "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi cat "${differencesFile}" >> "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi """ + env.buildStampFile = "${versionFolder}/buildstamp.txt" + sh """#!/bin/bash + + echo ${versionFolder}: >> "${OF}/buildstamps.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + cat "${buildStampFile}" >> "${OF}/buildstamps.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" totalrevision += intrevision echo "Accumulated revision is: ${totalrevision}" } - writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision + if (versionLabelPrefix.endsWith(".")) { + writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + totalrevision + } + else { + writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision + } } return this; \ No newline at end of file From 614a4b1e6b47f71c967780af75faa9090fe0fa1a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 13:37:17 +0100 Subject: [PATCH 014/195] Add metaversion generation --- test.gsh | 7 ------- versionnumbering/metaversion.gsh | 13 +++++-------- 2 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 test.gsh diff --git a/test.gsh b/test.gsh deleted file mode 100644 index 23fcfc0..0000000 --- a/test.gsh +++ /dev/null @@ -1,7 +0,0 @@ -// Methods in this file will end up as object methods on the object that load returns. -def lookAtThis(String whoAreYou) { - echo "Look at this, ${whoAreYou}! You loaded this from another file!" -} - -return this; - diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index e1cdaf6..aab7319 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,15 +33,12 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = cat "${differencesFile}" >> "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi """ - env.buildStampFile = "${versionFolder}/buildstamp.txt" - sh """#!/bin/bash + buildStampFile = "${versionFolder}/buildstamp.txt" + buildStampFileContents = readFile "${buildStampFile}" + metabuildStampFile = "${OF}/buildstamps.txt" + metabuildStampFileContents = readFile "${metabuildStampFile}" + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ": " + buildStampFileContents - echo ${versionFolder}: >> "${OF}/buildstamps.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - cat "${buildStampFile}" >> "${OF}/buildstamps.txt" - if [ \$? -ne 0 ]; then exit 1; fi - """ revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" From 4d9723e6155fb264bcebfcd44e5ba79740c58f07 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 27 Oct 2018 10:25:22 +0100 Subject: [PATCH 015/195] MCS compile should use build stamp generated by Common_scripts --- versionnumbering/versioninfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index b0d8ba1..62af192 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -13,6 +13,7 @@ import sys import shutil import subprocess import re +import datetime from argparse import ArgumentParser from argparse import RawDescriptionHelpFormatter @@ -151,6 +152,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: f.write(slaveMachine) + with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: + f.write(str(datetime.datetime.now)) + def main(argv=None): # IGNORE:C0111 '''Command line options.''' From 34e273f84a839d485344e24a372defea78d50054 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 27 Oct 2018 10:45:59 +0100 Subject: [PATCH 016/195] Add MCS jira key search --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 4937134..2d3ee0d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -78,6 +78,9 @@ jiraprojectpattern = { 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), ), + 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), + ), 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), From 476b4a59489df06cc15a6c29b7bf4ef514ef4d41 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 27 Oct 2018 10:56:49 +0100 Subject: [PATCH 017/195] Change base build number to match previous build system --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 62af192..1a6a72b 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -153,7 +153,7 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou f.write(slaveMachine) with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now)) + f.write(str(datetime.datetime.now())) def main(argv=None): # IGNORE:C0111 '''Command line options.''' From a441a0ba43586b6ff9db5c90d146e59f4e135f3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Oct 2018 00:24:11 +0100 Subject: [PATCH 018/195] Handle case where there only one version folder --- versioncontrol/git.gsh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index dbc09ff..904394d 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -43,9 +43,13 @@ String getSHAFromVersion(String versionFolder) { boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def keys = versions.keySet() - if (keys.size() <= 1) { + if (keys.size() < 1) { return false - } else { + } + if (keys.size() == 1) { + return true + } + else { version = "" for (key in keys) { nextversion = versions.get(key) From eaa8de01d5169f93d19b98699bd066e1650b4aea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 11:43:56 +0000 Subject: [PATCH 019/195] Move autobuild scripts into their own folder --- configuration/configuration.gsh | 2 +- versioncontrol/git.gsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index b91e0a3..9f60c0a 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -gh = load("${WORKSPACE}/Common_scripts/versioncontrol/git.gsh") +gh = load("${COMMON_SCRIPTS}/versioncontrol/git.gsh") configurationCollection = [:] diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 904394d..51dbd5d 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,4 +1,4 @@ -clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") +clh = load("${COMMON_SCRIPTS}/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 29d0f5e2ad63ff325ff9d5b2972b5d4b7b59f640 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 11:52:23 +0000 Subject: [PATCH 020/195] Move autobuild scripts into their own folder --- configuration/configuration.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 9f60c0a..5688482 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -8,7 +8,7 @@ def registerConfigurationInfo(String configurationName, String configurationURL, } def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${WORKSPACE}")) + registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${AUTOBUILD_SCRIPTS}")) } def saveConfigurationInVersionInfo() { From 8f9f29cad5badfa19290a59e3d7e13a9a197214a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 16:24:16 +0000 Subject: [PATCH 021/195] When working out what version prefix to use from a revision - if there are no dots (but there are numbers) default to 1.25000.0. --- versionnumbering/versioninfo.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 1a6a72b..ae2b0ee 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -76,6 +76,7 @@ def getVersionPrefix(sourcebranch): defaultversionprefix = "1.30000.0." mastertypeversionprefix = "1.20000.0." + thirdpartyversionprefix = "1.25000.0." splits = sourcebranch.split('/') if len(splits) < 1: print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) @@ -88,9 +89,16 @@ def getVersionPrefix(sourcebranch): splits2 = re.split(r'(\d+)', sourcebranch) splits = [] + hasdot = False for split in splits2: if split.isdigit(): splits.append(split) + if split == ".": + hasdot = True + + if not hasdot: + print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) + return thirdpartyversionprefix versionprefix = "" if len(splits) < 1: From bc79cbfbee40fd48d6857947ee657067e05b68b3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 16:52:36 +0000 Subject: [PATCH 022/195] Prevent revision number going above 16 bits worth of integer --- versionnumbering/versioninfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index ae2b0ee..7122dba 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -25,6 +25,7 @@ __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") slaveMachine = os.getenv("NODE_NAME", default="-") +MAXVERSIONNUMBER = 65535 def resetFolder(versionfolder): print("Removing folders and contents of: {0}".format(versionfolder)) @@ -48,7 +49,7 @@ def getGitRevision(gitrootpath, basenumber): print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) - adjustedchangecount = int(changecount) + int(basenumber) + adjustedchangecount = (int(changecount) + int(basenumber) % (MAXVERSIONNUMBER + 1)) print("Adjusted change count: {0}".format(adjustedchangecount)) print("Determining current revision info using git command: {0}".format(gitExecutable)) From 2376df47584eeb5da65593a09c30758a0cd5c4cc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 17:05:52 +0000 Subject: [PATCH 023/195] Prevent revision number going above 16 bits worth of integer --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 7122dba..ceae653 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -49,7 +49,7 @@ def getGitRevision(gitrootpath, basenumber): print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) - adjustedchangecount = (int(changecount) + int(basenumber) % (MAXVERSIONNUMBER + 1)) + adjustedchangecount = int((int(changecount) + int(basenumber)) % (MAXVERSIONNUMBER + 1)) print("Adjusted change count: {0}".format(adjustedchangecount)) print("Determining current revision info using git command: {0}".format(gitExecutable)) From b38a59a7dc77c58fff619112425a299498e0ee7a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 18:40:28 +0000 Subject: [PATCH 024/195] Move some installer helper functionality to Common_scripts --- install/installhelper.gsh | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 install/installhelper.gsh diff --git a/install/installhelper.gsh b/install/installhelper.gsh new file mode 100644 index 0000000..a55292d --- /dev/null +++ b/install/installhelper.gsh @@ -0,0 +1,41 @@ +// Given details about artifacts to import and details of installers to build, import artifacts that are required +def artifactsImporter(artifactDetails, installerDetails) { + artifactFetched = [:] + + for (installer in toBuild) { + def artifactKey = installerDetails.get(installer)[0] + //echo "To build: ${installer} use artifactKey: ${artifactKey}" + (sourceProject, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + env.importDest = destFolder + if (artifactFetched.get(artifactKey) == null) { + //echo "Artifact info: ${artifactInfo}" + copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" + def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" + def folderToCopy = "${destFolder}\\version" + echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" + fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) + //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) + artifactVersion = githelper.getSHAFromVersion("${destFolder}/version") + artifactFetched.put(artifactKey, artifactVersion) + } + } + return artifactFetched +} + +// Export any required binaries to the /bin folder +def exportBinaries(artifactDetails, installerDetails) { + for (installer in toBuild) { + (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + if (exportBinaries) { + (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + for (binaryFolder in binariesFolders) { + binariesSourceFolder = "${destinationFolder}/${binaryFolder}" + binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" + fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) + } + } + } +} + +return this; \ No newline at end of file From 28384f6ec3e8016f796f1e4fcbde28dce6d989ac Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 08:30:57 +0000 Subject: [PATCH 025/195] Only append differences to the metadifferences file if they exist --- versionnumbering/metaversion.gsh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index aab7319..9e09b34 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -24,15 +24,13 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = error "combineSubjobVersions script is not supported in Windows yet" } for (versionFolder in versionFolderList) { - env.differencesFile = "${versionFolder}/differences.txt" - sh """#!/bin/bash + differencesFile = "${versionFolder}/differences.txt" + if (fileExists("${differencesFile}")) { + currentContents = readFile file: "${OF}/differences.txt" + newContents = readFile file: "${differencesFile}" + writeFile file: "${OF}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + } - echo ${differencesFile}: >> "${OF}/differences.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - cat "${differencesFile}" >> "${OF}/differences.txt" - if [ \$? -ne 0 ]; then exit 1; fi - """ buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" metabuildStampFile = "${OF}/buildstamps.txt" From 53fd4d61ce4a19124c115aa017f984b0aa00cb0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 08:56:17 +0000 Subject: [PATCH 026/195] Add remote url to version information --- versionnumbering/versioninfo.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index ceae653..607ce96 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -64,10 +64,16 @@ def getGitRevision(gitrootpath, basenumber): remoteinfo = completed.stdout.decode("utf-8") print("Remote info: {0}".format(remoteinfo)) + print("Determining remote url using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "get-url", "--push", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteurl = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteurl)) + os.chdir(cwd) print("Restored current working directory to: {0}".format(cwd)) - return (adjustedchangecount, revision, remoteinfo) + return (adjustedchangecount, revision, remoteinfo, remoteurl) # Handle: # origin/master @@ -132,7 +138,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp): with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: f.write(buildjobname) @@ -148,6 +154,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: f.write(remoteinfo) + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: f.write(buildnumber) @@ -214,14 +223,14 @@ USAGE resetFolder(versionfolder) - (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) versionprefix = getVersionPrefix(sourcebranch) buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp) return 0 except Exception as e: From d842f1a87b9b3b2dd04002e1b0c9705a6d91c6ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 12:25:04 +0000 Subject: [PATCH 027/195] Output more git information in the meta version output --- versionnumbering/metaversion.gsh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 9e09b34..634da16 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,9 +33,13 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" + remoteurlFile = "${versionFolder}/remoteurl.txt" + remoteurlFileContents = readFile "${remoteurlFile}" + gitRevisionFile = "${versionFolder}/gitrevision.txt" + gitRevisionFileContents = readFile "${gitRevisionFile}" metabuildStampFile = "${OF}/buildstamps.txt" metabuildStampFileContents = readFile "${metabuildStampFile}" - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ": " + buildStampFileContents + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() From 469bf47a8cbbf070576bb2abf8e5d75ff4ee374c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 16:28:37 +0000 Subject: [PATCH 028/195] Handle the situation where a build does not provide a remoteurl.txt file --- versionnumbering/metaversion.gsh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 634da16..911e094 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,10 +33,18 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" + remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = readFile "${remoteurlFile}" + if (fileExists("${remoteurlFile}")) { + remoteurlFileContents = readFile "${remoteurlFile}" + } + else { + remoteurlFileContents = 'Unknown' + } + gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile "${gitRevisionFile}" + metabuildStampFile = "${OF}/buildstamps.txt" metabuildStampFileContents = readFile "${metabuildStampFile}" writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents From 75c7e069cccc27681245dfe06428a507cb9675be Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 20:28:30 +0000 Subject: [PATCH 029/195] Format metaversion output --- versionnumbering/metaversion.gsh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 911e094..d296fca 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -28,27 +28,27 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = if (fileExists("${differencesFile}")) { currentContents = readFile file: "${OF}/differences.txt" newContents = readFile file: "${differencesFile}" - writeFile file: "${OF}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + breakline = "***********************************************************************\n" + writeFile file: "${OF}/differences.txt", text: currentContents + breakline + "${differencesFile}: \n" + newContents } buildStampFile = "${versionFolder}/buildstamp.txt" - buildStampFileContents = readFile "${buildStampFile}" + buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" if (fileExists("${remoteurlFile}")) { - remoteurlFileContents = readFile "${remoteurlFile}" + remoteurlFileContents = readFile("${remoteurlFile}").trim() } else { remoteurlFileContents = 'Unknown' } gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile "${gitRevisionFile}" + gitRevisionFileContents = readFile("${gitRevisionFile}").trim() metabuildStampFile = "${OF}/buildstamps.txt" - metabuildStampFileContents = readFile "${metabuildStampFile}" + metabuildStampFileContents = readFile("${metabuildStampFile}") writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents - revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" From f6379b95dfd35b11455b6dd8b4432a58f68b1959 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 2 Nov 2018 17:36:15 +0000 Subject: [PATCH 030/195] Make up revision information from sub meta jobs --- versionnumbering/metaversion.gsh | 123 +++++++++++++++++++------------ 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index d296fca..cac6faf 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -1,66 +1,93 @@ +def mergemeta(versionFolder, outputFolder) { + differencesFile = "${versionFolder}/differences.txt" + currentContents = readFile file: "${outputFolder}/differences.txt" + newContents = readFile file: "${differencesFile}" + writeFile file: "${outputFolder}/differences.txt", text: currentContents + "\n" + newContents + + buildStampFile = "${versionFolder}/buildstamps.txt" + buildStampFileContents = readFile("${buildStampFile}").trim() + + metabuildStampFile = "${outputFolder}/buildstamps.txt" + metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\n" + "${buildStampFileContents}" + + if (fileExists("${versionFolder}/revision.txt")) { + revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + } + else { + intrevision = 1 + } + + return intrevision +} + +def appendcomponentinfo(versionFolder, outputFolder) { + differencesFile = "${versionFolder}/differences.txt" + if (fileExists("${differencesFile}")) { + currentContents = readFile file: "${outputFolder}/differences.txt" + newContents = readFile file: "${differencesFile}" + writeFile file: "${outputFolder}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + } + + buildStampFile = "${versionFolder}/buildstamp.txt" + buildStampFileContents = readFile("${buildStampFile}").trim() + + remoteurlFile = "${versionFolder}/remoteurl.txt" + if (fileExists("${remoteurlFile}")) { + remoteurlFileContents = readFile("${remoteurlFile}").trim() + } + else { + remoteurlFileContents = 'Unknown' + } + + gitRevisionFile = "${versionFolder}/gitrevision.txt" + gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + + metabuildStampFile = "${outputFolder}/buildstamps.txt" + if (fileExists("${differencesFile}")) { + metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" + } + else { + metabuildStampFileContents = "" + } + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + + revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + + return intrevision +} + // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = "${WORKSPACE}/metaversion") { - env.VLP = versionLabelPrefix - env.OF = outputFolder totalrevision = 0 - if (isUnix()) { - sh """#!/bin/bash - - rm -rf "${OF}" - mkdir -p "${OF}" - if [ \$? -ne 0 ]; then exit 1; fi - - touch "${OF}/differences.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - touch "${OF}/buildstamps.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - """ - } - else { - error "combineSubjobVersions script is not supported in Windows yet" + dir("${outputFolder}") { + deleteDir() } for (versionFolder in versionFolderList) { - differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - currentContents = readFile file: "${OF}/differences.txt" - newContents = readFile file: "${differencesFile}" - breakline = "***********************************************************************\n" - writeFile file: "${OF}/differences.txt", text: currentContents + breakline + "${differencesFile}: \n" + newContents - } - - buildStampFile = "${versionFolder}/buildstamp.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() - - remoteurlFile = "${versionFolder}/remoteurl.txt" - if (fileExists("${remoteurlFile}")) { - remoteurlFileContents = readFile("${remoteurlFile}").trim() + metaversionsignaturefile = "${versionFolder}/metaversion.txt" + // If the sub project is a meta job + if (fileExists("${metaversionsignaturefile}")) { + intrevision = mergemeta("${versionFolder}", outputFolder) } else { - remoteurlFileContents = 'Unknown' + intrevision = appendcomponentinfo("${versionFolder}", outputFolder) } - - gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - metabuildStampFile = "${OF}/buildstamps.txt" - metabuildStampFileContents = readFile("${metabuildStampFile}") - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents - revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - totalrevision += intrevision - echo "Accumulated revision is: ${totalrevision}" } + totalrevision += intrevision + echo "Accumulated revision is: ${totalrevision}" if (versionLabelPrefix.endsWith(".")) { - writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + totalrevision + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + totalrevision } else { - writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } + writeFile("${outputFolder}/revision.txt").trim(), text: totalrevision) } return this; \ No newline at end of file From 67895110b66406b57385835d80888ecfbd8d9c2a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 4 Nov 2018 17:34:35 +0000 Subject: [PATCH 031/195] Make up revision information from sub meta jobs --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index cac6faf..4a821be 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -87,7 +87,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = else { writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } - writeFile("${outputFolder}/revision.txt").trim(), text: totalrevision) + writeFile("${outputFolder}/revision.txt", text: totalrevision) } return this; \ No newline at end of file From dc3abfc7cf5e4823af8e22da5a83b28de8671721 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 4 Nov 2018 20:46:58 +0000 Subject: [PATCH 032/195] More debug; Common functions --- versionnumbering/metaversion.gsh | 43 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 4a821be..8849ae3 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -1,16 +1,26 @@ +def currentContents(file, newline = "\n", defaultContents = "") { + if (fileExists("${file}")) { + contents = readFile(file: "${file}").trim() + newline + } + else { + contents = defaultContents + } + return contents +} + def mergemeta(versionFolder, outputFolder) { + echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" + differencesFile = "${versionFolder}/differences.txt" - currentContents = readFile file: "${outputFolder}/differences.txt" newContents = readFile file: "${differencesFile}" - writeFile file: "${outputFolder}/differences.txt", text: currentContents + "\n" + newContents + metadifferencesFile = ${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents buildStampFile = "${versionFolder}/buildstamps.txt" buildStampFileContents = readFile("${buildStampFile}").trim() - metabuildStampFile = "${outputFolder}/buildstamps.txt" - metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\n" + "${buildStampFileContents}" + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents if (fileExists("${versionFolder}/revision.txt")) { revision = readFile("${versionFolder}/revision.txt").trim() @@ -25,36 +35,26 @@ def mergemeta(versionFolder, outputFolder) { } def appendcomponentinfo(versionFolder, outputFolder) { + echo "Appemding component version data from folder: ${versionFolder} to ${outputFolder}" + differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { - currentContents = readFile file: "${outputFolder}/differences.txt" newContents = readFile file: "${differencesFile}" - writeFile file: "${outputFolder}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" - if (fileExists("${remoteurlFile}")) { - remoteurlFileContents = readFile("${remoteurlFile}").trim() - } - else { - remoteurlFileContents = 'Unknown' - } + remoteurlFileContents = currentContents(remoteurlFile. "", 'Unknown') gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile("${gitRevisionFile}").trim() metabuildStampFile = "${outputFolder}/buildstamps.txt" - if (fileExists("${differencesFile}")) { - metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" - } - else { - metabuildStampFileContents = "" - } - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents - + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" @@ -70,6 +70,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = deleteDir() } for (versionFolder in versionFolderList) { + echo "Inspecting folder: ${versionFolder}" metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job if (fileExists("${metaversionsignaturefile}")) { From 3a5980aa9823dba95537bfd0b3e207266e5bd108 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 4 Nov 2018 23:20:10 +0000 Subject: [PATCH 033/195] More debug; Common functions --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 8849ae3..6e57d4c 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -14,7 +14,7 @@ def mergemeta(versionFolder, outputFolder) { differencesFile = "${versionFolder}/differences.txt" newContents = readFile file: "${differencesFile}" - metadifferencesFile = ${outputFolder}/differences.txt" + metadifferencesFile = "${outputFolder}/differences.txt" writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents buildStampFile = "${versionFolder}/buildstamps.txt" From 6c617020f10ae654e423f54a70790552123501ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 11:55:54 +0000 Subject: [PATCH 034/195] Accumulate the total revision --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 6e57d4c..61f8a3b 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -79,8 +79,8 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = else { intrevision = appendcomponentinfo("${versionFolder}", outputFolder) } + totalrevision += intrevision } - totalrevision += intrevision echo "Accumulated revision is: ${totalrevision}" if (versionLabelPrefix.endsWith(".")) { writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + totalrevision From 2f75cd2d14ef5a9836be3554a04fe20dcc6c16ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 14:32:04 +0000 Subject: [PATCH 035/195] Handle syntax error --- versionnumbering/metaversion.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 61f8a3b..22168ba 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -35,7 +35,7 @@ def mergemeta(versionFolder, outputFolder) { } def appendcomponentinfo(versionFolder, outputFolder) { - echo "Appemding component version data from folder: ${versionFolder} to ${outputFolder}" + echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { @@ -48,7 +48,7 @@ def appendcomponentinfo(versionFolder, outputFolder) { buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = currentContents(remoteurlFile. "", 'Unknown') + remoteurlFileContents = currentContents(remoteurlFile, "", 'Unknown').trim() gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile("${gitRevisionFile}").trim() From b6d7567c364668f072789725ef3b84523442f4bd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 15:06:41 +0000 Subject: [PATCH 036/195] Handle a release to git job name parameter - to be passed to metabuilder jobs --- versionnumbering/versioninfo.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 607ce96..370d9fc 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -138,7 +138,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp): +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: f.write(buildjobname) @@ -173,6 +173,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: f.write(str(datetime.datetime.now())) + with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: + f.write(releasejobname) + def main(argv=None): # IGNORE:C0111 '''Command line options.''' @@ -206,6 +209,7 @@ USAGE parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + parser.add_argument("-r", "--releasejobname", dest="releasejobname", action="store", default="-", help="release-to-git build job name [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -217,9 +221,10 @@ USAGE baseversionnumber = args.baseversionnumber buildjobname = args.buildjobname buildnumber = args.buildnumber + releasejobname = args.releasejobname print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber, "Release job name:", releasejobname) resetFolder(versionfolder) @@ -230,7 +235,7 @@ USAGE buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp) + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname) return 0 except Exception as e: From a6511a91beb44ab82ae292fd67fd9a96ddbfb3a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 15:47:49 +0000 Subject: [PATCH 037/195] Syntax error --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 22168ba..084b604 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -88,7 +88,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = else { writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } - writeFile("${outputFolder}/revision.txt", text: totalrevision) + writeFile(file: "${outputFolder}/revision.txt", text: totalrevision) } return this; \ No newline at end of file From 6fa63d47f3e796b00befcad58e8bdc76497462ef Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 16:19:52 +0000 Subject: [PATCH 038/195] Add release job name to metabuildstamp file --- versionnumbering/metaversion.gsh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 084b604..1dc19ca 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -53,8 +53,11 @@ def appendcomponentinfo(versionFolder, outputFolder) { gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + releaseJobNameFile = "${versionFolder}/releasejobname.txt" + releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() + metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" From dcdf1d3dc3a866d3cb73f3ce9f577105c6e89d3d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 19:00:51 +0000 Subject: [PATCH 039/195] Correct integer printing --- versionnumbering/metaversion.gsh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 1dc19ca..9a50e2e 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -85,13 +85,8 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = totalrevision += intrevision } echo "Accumulated revision is: ${totalrevision}" - if (versionLabelPrefix.endsWith(".")) { - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + totalrevision - } - else { - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision - } - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision) + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() + writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) } return this; \ No newline at end of file From 0c2b471c3cea162ff6169768e2c9da8eabc4498d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 7 Nov 2018 22:36:03 +0000 Subject: [PATCH 040/195] Only build selected variants --- compile/eclipsebuild/eclipseds5build.gsh | 48 +++++++++++++++--------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 3061fea..078845e 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,14 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild. exportfolder = "${WORKSPACE}/bin") { + toBuild = [] + if (variantstobuild) { + toBuildUntrimmed = variantstobuild.split(',') + toBuildUntrimmed.each { + toBuild.add(it.trim()) + } + } // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { try { @@ -137,26 +144,31 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) try { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } } catch (Exception e) { echo "Build failed - Waiting for a while" From 9f1d3ca026847e066d3fe9921aa68a2b99812a80 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 7 Nov 2018 22:39:15 +0000 Subject: [PATCH 041/195] Only build selected variants --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 078845e..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,7 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild. exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}/bin") { toBuild = [] if (variantstobuild) { toBuildUntrimmed = variantstobuild.split(',') From 69de014d49076c0d25a24fe04600fedbc29e110f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:04:34 +0000 Subject: [PATCH 042/195] Condense autobuild startup boilerplate --- autobuilder.gsh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 autobuilder.gsh diff --git a/autobuilder.gsh b/autobuilder.gsh new file mode 100644 index 0000000..06a5123 --- /dev/null +++ b/autobuilder.gsh @@ -0,0 +1,39 @@ +// Assume that WORKSPACE environment variable is set + +def autobuildroot = "${WORKSPACE}/autobuild" +def commonscriptsroot = "${autobuildroot}/Common_scripts" + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + cleanWs() + unstash name: 'Clean workspace stash' + } + else { + echo "Not cleaning workspace" + } +} + +// Start up boilderplate +def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autobuildroot}/Common_scripts") { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") + configurationhelper.registerPipelineScriptConfiguration() + +def commonscripts() + return commonscriptsroot +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.gsh") +} + +return this; \ No newline at end of file From 3e7191297956761eead46374cc6cfd7348a97136 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:08:35 +0000 Subject: [PATCH 043/195] Condense autobuild startup boilerplate --- autobuilder.gsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 06a5123..7a7efa1 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -27,8 +27,9 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") configurationhelper.registerPipelineScriptConfiguration() +} -def commonscripts() +def commonscripts() { return commonscriptsroot } From 12f680285f37934d9dc4ff56b7b961149febbc2f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:13:21 +0000 Subject: [PATCH 044/195] Condense autobuild startup boilerplate --- autobuilder.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 7a7efa1..1657815 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,7 +1,7 @@ // Assume that WORKSPACE environment variable is set -def autobuildroot = "${WORKSPACE}/autobuild" -def commonscriptsroot = "${autobuildroot}/Common_scripts" +String autobuildroot = "${WORKSPACE}/autobuild" +String commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From a870eb3e57496a1ff78e182fc9cb543d2f50bfd0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:23:19 +0000 Subject: [PATCH 045/195] Condense autobuild startup boilerplate --- autobuilder.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 1657815..88b68a0 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,7 +1,10 @@ // Assume that WORKSPACE environment variable is set -String autobuildroot = "${WORKSPACE}/autobuild" -String commonscriptsroot = "${autobuildroot}/Common_scripts" +@Field +def autobuildroot = "${WORKSPACE}/autobuild" + +@Field +def commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 16c6e92ac9beaa917a8acde5802a491eba34b27e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:25:18 +0000 Subject: [PATCH 046/195] Condense autobuild startup boilerplate --- autobuilder.gsh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 88b68a0..bf42e4c 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,10 +1,8 @@ // Assume that WORKSPACE environment variable is set -@Field -def autobuildroot = "${WORKSPACE}/autobuild" +autobuildroot = "${WORKSPACE}/autobuild" -@Field -def commonscriptsroot = "${autobuildroot}/Common_scripts" +commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 9403f706aba7c1147d2ddce2a119edd6bae9b54b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:31:25 +0000 Subject: [PATCH 047/195] Condense autobuild startup boilerplate --- configuration/configuration.gsh | 2 +- versioncontrol/git.gsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 5688482..dbaf921 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -gh = load("${COMMON_SCRIPTS}/versioncontrol/git.gsh") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.gsh") configurationCollection = [:] diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 51dbd5d..1462a8e 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,4 +1,4 @@ -clh = load("${COMMON_SCRIPTS}/commandline/commandline.gsh") +clh = load("${autobuilder.commonscripts()}/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 3f3ee1c70997abd6693ea0c5224bbb8ec99f57bf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:37:04 +0000 Subject: [PATCH 048/195] Condense autobuild startup boilerplate --- autobuilder.gsh | 4 ++++ configuration/configuration.gsh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index bf42e4c..6ec0e38 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -34,6 +34,10 @@ def commonscripts() { return commonscriptsroot } +def getautobuildpath() { + return autobuildroot +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.gsh") } diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index dbaf921..f131c98 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -8,7 +8,7 @@ def registerConfigurationInfo(String configurationName, String configurationURL, } def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${AUTOBUILD_SCRIPTS}")) + registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } def saveConfigurationInVersionInfo() { From 94941c199f983f328906b85c02394fc77fb42326 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:46:24 +0000 Subject: [PATCH 049/195] Change .gsh suffix to .groovy --- artifacts/{metaexport.gsh => metaexport.groovy} | 0 autobuilder.gsh => autobuilder.groovy | 0 commandline/{commandline.gsh => commandline.groovy} | 0 .../{disableparallelbuilds.gsh => disableparallelbuilds.groovy} | 0 .../eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} | 0 configuration/{configuration.gsh => configuration.groovy} | 0 differences/{json.gsh => json.groovy} | 0 install/{installhelper.gsh => installhelper.groovy} | 0 versioncontrol/{git.gsh => git.groovy} | 0 versionnumbering/{metaversion.gsh => metaversion.groovy} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename artifacts/{metaexport.gsh => metaexport.groovy} (100%) rename autobuilder.gsh => autobuilder.groovy (100%) rename commandline/{commandline.gsh => commandline.groovy} (100%) rename compile/eclipsebuild/{disableparallelbuilds.gsh => disableparallelbuilds.groovy} (100%) rename compile/eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} (100%) rename configuration/{configuration.gsh => configuration.groovy} (100%) rename differences/{json.gsh => json.groovy} (100%) rename install/{installhelper.gsh => installhelper.groovy} (100%) rename versioncontrol/{git.gsh => git.groovy} (100%) rename versionnumbering/{metaversion.gsh => metaversion.groovy} (100%) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.groovy similarity index 100% rename from artifacts/metaexport.gsh rename to artifacts/metaexport.groovy diff --git a/autobuilder.gsh b/autobuilder.groovy similarity index 100% rename from autobuilder.gsh rename to autobuilder.groovy diff --git a/commandline/commandline.gsh b/commandline/commandline.groovy similarity index 100% rename from commandline/commandline.gsh rename to commandline/commandline.groovy diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.groovy similarity index 100% rename from compile/eclipsebuild/disableparallelbuilds.gsh rename to compile/eclipsebuild/disableparallelbuilds.groovy diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.groovy similarity index 100% rename from compile/eclipsebuild/eclipseds5build.gsh rename to compile/eclipsebuild/eclipseds5build.groovy diff --git a/configuration/configuration.gsh b/configuration/configuration.groovy similarity index 100% rename from configuration/configuration.gsh rename to configuration/configuration.groovy diff --git a/differences/json.gsh b/differences/json.groovy similarity index 100% rename from differences/json.gsh rename to differences/json.groovy diff --git a/install/installhelper.gsh b/install/installhelper.groovy similarity index 100% rename from install/installhelper.gsh rename to install/installhelper.groovy diff --git a/versioncontrol/git.gsh b/versioncontrol/git.groovy similarity index 100% rename from versioncontrol/git.gsh rename to versioncontrol/git.groovy diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.groovy similarity index 100% rename from versionnumbering/metaversion.gsh rename to versionnumbering/metaversion.groovy From 76913ba017012d451f6681d6bf44704178515ae9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:54:02 +0000 Subject: [PATCH 050/195] Change .gsh suffix to .groovy --- autobuilder.groovy | 4 ++-- configuration/configuration.groovy | 2 +- versioncontrol/git.groovy | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 6ec0e38..54cfb12 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob conditionalclean() - configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") + configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") configurationhelper.registerPipelineScriptConfiguration() } @@ -39,7 +39,7 @@ def getautobuildpath() { } def configurationhelper() { - return load("${commonscripts()}/configuration/configuration.gsh") + return load("${commonscripts()}/configuration/configuration.groovy") } return this; \ No newline at end of file diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index f131c98..1f0c095 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,4 +1,4 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.gsh") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") configurationCollection = [:] diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1462a8e..031094b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,4 +1,4 @@ -clh = load("${autobuilder.commonscripts()}/commandline/commandline.gsh") +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From ad68c4caf5f990cca4a774d46dac162666d1817d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 22:35:36 +0000 Subject: [PATCH 051/195] Remove clean boilerplate --- clean/clean.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 clean/clean.groovy diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file From e3533bd5f140a1561520b416ef585f366ed9dca4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 23:29:30 +0000 Subject: [PATCH 052/195] Remove versionnumbering boilerplate --- versionnumbering/generate.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 versionnumbering/generate.groovy diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..6e87392 --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,20 @@ + +def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${WORKSPACE}/version", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { + cs = autobuilder.commonscripts() + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${cs}%\\versionnumbering\\versioninfo.py" -s "%${sourcerevision}%" -v "${versionfolder}" -g "${sourcepath}" -b %${basebuildnumber}% -j "%${jobname}%" -n "%${buildid}%" -r "%${releasejobname}%" + """ + } + env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() + env.BUILD_STAMP_COMMA_SEP = readFile("${WORKSPACE}/version/buildstampcommasep.txt").trim() + env.SRC_REVISION = readFile("${WORKSPACE}/version/revision.txt").trim() + configurationhelper.saveConfigurationInVersionInfo(); +} + +return this; \ No newline at end of file From b0077bb1df56c27f18921cd7184d6e939fa43281 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 23:49:04 +0000 Subject: [PATCH 053/195] Remove differences boilerplate --- differences/do.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 differences/do.groovy diff --git a/differences/do.groovy b/differences/do.groovy new file mode 100644 index 0000000..9dc1807 --- /dev/null +++ b/differences/do.groovy @@ -0,0 +1,17 @@ + +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { + cs = autobuilder.commonscripts() + + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${cs}%\\differences\\show_release_differences.py" -g "%${sourcepath}%" -s "%${sourcerevision}%" -p "%${previoussourcerevision}%" -j ${jirakey} -o "opposite notcommon" -v "%${versionfolder}%" + """ + } +} + +return this; From 1d7346e5a02c0018986e56a47987adf197874731 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 11 Nov 2018 11:43:28 +0000 Subject: [PATCH 054/195] Remove boilerplate --- versionnumbering/do.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 versionnumbering/do.groovy diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..9b4a42e --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,20 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + filetoconvert = convertdetail[0] + pattern = convertdetail[1] + replacement = convertdetail[2] + if (isUnix()) { + sh """ + "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + """ + } + } +} + +return this; From b92a8261897b30807c06198982d92a5db1cfd805 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 12 Nov 2018 10:50:37 +0000 Subject: [PATCH 055/195] Rename .gsh files to .groovy --- artifacts/{metaexport.gsh => metaexport.groovy} | 0 commandline/{commandline.gsh => commandline.groovy} | 0 .../{disableparallelbuilds.gsh => disableparallelbuilds.groovy} | 0 .../eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} | 0 configuration/{configuration.gsh => configuration.groovy} | 0 differences/{json.gsh => json.groovy} | 0 install/{installhelper.gsh => installhelper.groovy} | 0 versioncontrol/{git.gsh => git.groovy} | 0 versionnumbering/{metaversion.gsh => metaversion.groovy} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename artifacts/{metaexport.gsh => metaexport.groovy} (100%) rename commandline/{commandline.gsh => commandline.groovy} (100%) rename compile/eclipsebuild/{disableparallelbuilds.gsh => disableparallelbuilds.groovy} (100%) rename compile/eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} (100%) rename configuration/{configuration.gsh => configuration.groovy} (100%) rename differences/{json.gsh => json.groovy} (100%) rename install/{installhelper.gsh => installhelper.groovy} (100%) rename versioncontrol/{git.gsh => git.groovy} (100%) rename versionnumbering/{metaversion.gsh => metaversion.groovy} (100%) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.groovy similarity index 100% rename from artifacts/metaexport.gsh rename to artifacts/metaexport.groovy diff --git a/commandline/commandline.gsh b/commandline/commandline.groovy similarity index 100% rename from commandline/commandline.gsh rename to commandline/commandline.groovy diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.groovy similarity index 100% rename from compile/eclipsebuild/disableparallelbuilds.gsh rename to compile/eclipsebuild/disableparallelbuilds.groovy diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.groovy similarity index 100% rename from compile/eclipsebuild/eclipseds5build.gsh rename to compile/eclipsebuild/eclipseds5build.groovy diff --git a/configuration/configuration.gsh b/configuration/configuration.groovy similarity index 100% rename from configuration/configuration.gsh rename to configuration/configuration.groovy diff --git a/differences/json.gsh b/differences/json.groovy similarity index 100% rename from differences/json.gsh rename to differences/json.groovy diff --git a/install/installhelper.gsh b/install/installhelper.groovy similarity index 100% rename from install/installhelper.gsh rename to install/installhelper.groovy diff --git a/versioncontrol/git.gsh b/versioncontrol/git.groovy similarity index 100% rename from versioncontrol/git.gsh rename to versioncontrol/git.groovy diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.groovy similarity index 100% rename from versionnumbering/metaversion.gsh rename to versionnumbering/metaversion.groovy From 4f62592e7f7bf0c91cb372f12d10c83fa208d4c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 13 Nov 2018 09:39:25 +0000 Subject: [PATCH 056/195] buildstamp.txt -> buildstamps.txt --- versionnumbering/metaversion.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 9a50e2e..e566aa1 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -44,7 +44,7 @@ def appendcomponentinfo(versionFolder, outputFolder) { writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } - buildStampFile = "${versionFolder}/buildstamp.txt" + buildStampFile = "${versionFolder}/buildstamps.txt" buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" From 2b616e5397f88d7acbbdca593db7a7ce9ecae45f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:21:52 +0000 Subject: [PATCH 057/195] Handle version numbering in windows --- versionnumbering/generate.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 6e87392..cf02860 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -8,7 +8,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } else { bat """ - "%${PYTHON}%" -u "%${cs}%\\versionnumbering\\versioninfo.py" -s "%${sourcerevision}%" -v "${versionfolder}" -g "${sourcepath}" -b %${basebuildnumber}% -j "%${jobname}%" -n "%${buildid}%" -r "%${releasejobname}%" + \"${PYTHON}\" -u \"${cs}\\versionnumbering\\versioninfo.py\" -s \"${sourcerevision}\" -v \"${versionfolder}\" -g \"${sourcepath}\" -b ${basebuildnumber} -j \"${jobname}\" -n \"${buildid}\" -r \"${releasejobname}\" """ } env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() @@ -17,4 +17,4 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam configurationhelper.saveConfigurationInVersionInfo(); } -return this; \ No newline at end of file +return this; From 8ce03b4e10a7016e46bfd9f1f91c30822fdc89ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:28:10 +0000 Subject: [PATCH 058/195] Handle version numbering in windows --- differences/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/do.groovy b/differences/do.groovy index 9dc1807..7279c0b 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -9,7 +9,7 @@ def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, ver } else { bat """ - "%${PYTHON}%" -u "%${cs}%\\differences\\show_release_differences.py" -g "%${sourcepath}%" -s "%${sourcerevision}%" -p "%${previoussourcerevision}%" -j ${jirakey} -o "opposite notcommon" -v "%${versionfolder}%" + \"${PYTHON}\" -u \"${cs}\\differences\\show_release_differences.py\" -g \"${sourcepath}\" -s \"${sourcerevision}\" -p \"${previoussourcerevision}\" -j ${jirakey} -o \"opposite notcommon\" -v \"${versionfolder}\" """ } } From 143670fbc6aba03871871b44aae0e7cd6d5b6ffe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:44:11 +0000 Subject: [PATCH 059/195] Handle version numbering in windows --- differences/do.groovy | 9 +++------ versionnumbering/generate.groovy | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 7279c0b..d0a489b 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,15 +2,12 @@ def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() + diffcmd = '"${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}"' if (isUnix()) { - sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" - """ + sh "${diffcmd}" } else { - bat """ - \"${PYTHON}\" -u \"${cs}\\differences\\show_release_differences.py\" -g \"${sourcepath}\" -s \"${sourcerevision}\" -p \"${previoussourcerevision}\" -j ${jirakey} -o \"opposite notcommon\" -v \"${versionfolder}\" - """ + bat "${diffcmd}" } } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cf02860..2776130 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -8,7 +8,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } else { bat """ - \"${PYTHON}\" -u \"${cs}\\versionnumbering\\versioninfo.py\" -s \"${sourcerevision}\" -v \"${versionfolder}\" -g \"${sourcepath}\" -b ${basebuildnumber} -j \"${jobname}\" -n \"${buildid}\" -r \"${releasejobname}\" + "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" """ } env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() From 4c737131d55369f5deb9d2919c9632b401bea703 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:45:05 +0000 Subject: [PATCH 060/195] Handle version numbering in windows --- differences/do.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index d0a489b..b3039a9 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,12 +2,15 @@ def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() - diffcmd = '"${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}"' if (isUnix()) { - sh "${diffcmd}" + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ } else { - bat "${diffcmd}" + bat """ + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ } } From bf1cba0f8baccaa4c5ea3bbe8ee60f870784a4ba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 15 Nov 2018 23:53:08 +0000 Subject: [PATCH 061/195] Prefix each line of a metaversion file with path to relevant artifact --- versionnumbering/metaversion.groovy | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index e566aa1..00256cc 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -9,7 +9,16 @@ def currentContents(file, newline = "\n", defaultContents = "") { return contents } -def mergemeta(versionFolder, outputFolder) { +def prefixline(rootpath, filepath) { + fileContents = readFile("${filepath}").trim() + newFileContents = "" + fileContents.eachline { + newFileContents += "${rootpath}" + "/" + it + } + return newFileContents +} + +def mergemeta(rootpath, versionFolder, outputFolder) { echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" differencesFile = "${versionFolder}/differences.txt" @@ -18,7 +27,7 @@ def mergemeta(versionFolder, outputFolder) { writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() + buildStampFileContents = prefixline(rootpath, "${buildStampFile}") metabuildStampFile = "${outputFolder}/buildstamps.txt" writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents @@ -34,7 +43,7 @@ def mergemeta(versionFolder, outputFolder) { return intrevision } -def appendcomponentinfo(versionFolder, outputFolder) { +def appendcomponentinfo(rootpath, versionFolder, outputFolder) { echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" differencesFile = "${versionFolder}/differences.txt" @@ -57,7 +66,7 @@ def appendcomponentinfo(versionFolder, outputFolder) { releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" @@ -67,20 +76,21 @@ def appendcomponentinfo(versionFolder, outputFolder) { // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = "${WORKSPACE}/metaversion") { +def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { totalrevision = 0 dir("${outputFolder}") { deleteDir() } - for (versionFolder in versionFolderList) { + for (artifactItem in artifactList) { + (rootpath, versionFolder) = artifactItem echo "Inspecting folder: ${versionFolder}" metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta("${versionFolder}", outputFolder) + intrevision = mergemeta(rootpath, versionFolder, outputFolder) } else { - intrevision = appendcomponentinfo("${versionFolder}", outputFolder) + intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) } totalrevision += intrevision } From 7114a1bf1c98f95d8d51b3c76176585a21d594d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 13:43:05 +0000 Subject: [PATCH 062/195] Remove some scope of variables being created --- autobuilder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 54cfb12..eb32dfc 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,8 +1,8 @@ // Assume that WORKSPACE environment variable is set -autobuildroot = "${WORKSPACE}/autobuild" +def autobuildroot = "${WORKSPACE}/autobuild" -commonscriptsroot = "${autobuildroot}/Common_scripts" +def commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { @@ -26,8 +26,8 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob conditionalclean() - configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") - configurationhelper.registerPipelineScriptConfiguration() + def ch = load("${commonscriptsroot}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration() } def commonscripts() { From 1552da9e07db5be1e461548d349d6844a81e92dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 13:50:31 +0000 Subject: [PATCH 063/195] Remove some scope of variables being created --- autobuilder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index eb32dfc..f07e945 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,8 +1,8 @@ // Assume that WORKSPACE environment variable is set -def autobuildroot = "${WORKSPACE}/autobuild" +autobuildroot = "${WORKSPACE}/autobuild" -def commonscriptsroot = "${autobuildroot}/Common_scripts" +commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 09a1c4fa507de3b417a2612fe80d5ea9cedc94f8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 14:04:56 +0000 Subject: [PATCH 064/195] Locate required githelper script --- install/installhelper.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index a55292d..f523799 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -15,7 +15,8 @@ def artifactsImporter(artifactDetails, installerDetails) { echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) - artifactVersion = githelper.getSHAFromVersion("${destFolder}/version") + def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + artifactVersion = gh.getSHAFromVersion("${destFolder}/version") artifactFetched.put(artifactKey, artifactVersion) } } From fdff4d689313b5cf358ca33db588cb2fcbe316c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 23:29:39 +0000 Subject: [PATCH 065/195] Correct file line handling --- versionnumbering/metaversion.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 00256cc..3b8d7c1 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -10,10 +10,13 @@ def currentContents(file, newline = "\n", defaultContents = "") { } def prefixline(rootpath, filepath) { - fileContents = readFile("${filepath}").trim() - newFileContents = "" - fileContents.eachline { - newFileContents += "${rootpath}" + "/" + it + def fileContents = readFile("${filepath}").trim() + def fileContentsNormalized = fileContents.normalize() + + def newFileContents = "" + def lines = fileContentsNormalized.readLines() + for (aline in lines) { + newFileContents += "${rootpath}" + "/" + aline } return newFileContents } From 5ee5219418ae4de60c76db138ac8b912365af0b9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 11:36:47 +0000 Subject: [PATCH 066/195] Add platform specific line endings definition --- autobuilder.groovy | 4 +++- platform/lineendings.groovy | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 platform/lineendings.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 54cfb12..ef4f938 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,6 +28,8 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") configurationhelper.registerPipelineScriptConfiguration() + + lineendings = load("${commonscriptsroot}/platform/lineendings.groovy") } def commonscripts() { @@ -42,4 +44,4 @@ def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } -return this; \ No newline at end of file +return this; diff --git a/platform/lineendings.groovy b/platform/lineendings.groovy new file mode 100644 index 0000000..9657b0d --- /dev/null +++ b/platform/lineendings.groovy @@ -0,0 +1,5 @@ + +// Put ${nl} into strings when wanting a newline +nl = System.properties['line.separator'] + +return this; From ce133e9ba9ca4812c0afa08a6c5fb60aef42f3ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 11:55:39 +0000 Subject: [PATCH 067/195] Put line endings on metaversion file --- versionnumbering/metaversion.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 3b8d7c1..1f6cff6 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -1,4 +1,6 @@ +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") + def currentContents(file, newline = "\n", defaultContents = "") { if (fileExists("${file}")) { contents = readFile(file: "${file}").trim() + newline @@ -16,7 +18,7 @@ def prefixline(rootpath, filepath) { def newFileContents = "" def lines = fileContentsNormalized.readLines() for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + newFileContents += "${rootpath}" + "/" + aline + "${nl}" } return newFileContents } From 18cc8e0609829df0468474e9c2e70285d6ec12d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 20:25:15 +0000 Subject: [PATCH 068/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 31 ++++++++++++++++ file/utils.groovy | 16 ++++++++ publishing/utils.groovy | 25 +++++++++++++ versionnumbering/generate.groovy | 11 ++++-- versionnumbering/metaversion.groovy | 57 +++++++++++++++-------------- 5 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 checksum/utils.groovy create mode 100644 file/utils.groovy create mode 100644 publishing/utils.groovy diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..b1a714f --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,31 @@ + +env.CHECKSUMARTIFACTS = "checksums/**" +def CHECKSUMFOLDER = "${WORKSPACE}/checksums" + +// SHA256SUM environment variable must be set to sha256sum utility +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(checksummable) { + if !fileExists("${SHA256SUM}") { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + } + sh """ + rm -rf "${CHECKSUMFOLDER}" + mkdir -p "${CHECKSUMFOLDER}" + """ + for (filepath in checksummable) { + def files = findFiles(glob: filepath) + for (file in files) { + if isUnix() { + sh """#!/bin/bash + "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + """ + } + else { + bat """ + "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + """ + } + } + } +} + diff --git a/file/utils.groovy b/file/utils.groovy new file mode 100644 index 0000000..0ffa47c --- /dev/null +++ b/file/utils.groovy @@ -0,0 +1,16 @@ + +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") + +// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents +def currentContents(file, newline = "${nl}", defaultContents = "") { + def contents = "" + if (fileExists("${file}")) { + contents = readFile(file: "${file}").trim() + newline + } + else { + contents = defaultContents + } + return contents +} + +return this; diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..a10fd76 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,25 @@ + +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") + +env.PUBLISHABLEARTIFACTS = "publishable/**" +def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" +def PUBLISHABLEFILE = "${WORKSPACE}/publishable/list.json" + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(publishable) { + sh """ + rm -rf "${PUBLISHABLEFOLDER}" + mkdir -p "${PUBLISHABLEFOLDER}" + """ + filelist = [] + for (filepath in publishable) { + def files = findFiles(glob: filepath) + for (file in files) { + if (!file.directory) { + filelist.add(file.path) + } + } + } + def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) + writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) +} \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 2776130..6f9f803 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,5 +1,8 @@ -def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${WORKSPACE}/version", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { +env.VERSIONARTIFACTS = "version/**" +def VERSIONFOLDER = "${WORKSPACE}/version" + +def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { cs = autobuilder.commonscripts() if (isUnix()) { sh """ @@ -11,9 +14,9 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" """ } - env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() - env.BUILD_STAMP_COMMA_SEP = readFile("${WORKSPACE}/version/buildstampcommasep.txt").trim() - env.SRC_REVISION = readFile("${WORKSPACE}/version/revision.txt").trim() + env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() + env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() + env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() configurationhelper.saveConfigurationInVersionInfo(); } diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 1f6cff6..ff20a86 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -1,5 +1,6 @@ def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") def currentContents(file, newline = "\n", defaultContents = "") { if (fileExists("${file}")) { @@ -26,18 +27,19 @@ def prefixline(rootpath, filepath) { def mergemeta(rootpath, versionFolder, outputFolder) { echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - differencesFile = "${versionFolder}/differences.txt" - newContents = readFile file: "${differencesFile}" - metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents + def differencesFile = "${versionFolder}/differences.txt" + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents + def intrevision = 0 if (fileExists("${versionFolder}/revision.txt")) { - revision = readFile("${versionFolder}/revision.txt").trim() + def revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" } @@ -51,29 +53,29 @@ def mergemeta(rootpath, versionFolder, outputFolder) { def appendcomponentinfo(rootpath, versionFolder, outputFolder) { echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - differencesFile = "${versionFolder}/differences.txt" + def differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { - newContents = readFile file: "${differencesFile}" - metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } - buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = readFile("${buildStampFile}").trim() - remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = currentContents(remoteurlFile, "", 'Unknown').trim() + def remoteurlFile = "${versionFolder}/remoteurl.txt" + def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + def gitRevisionFile = "${versionFolder}/gitrevision.txt" + def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - releaseJobNameFile = "${versionFolder}/releasejobname.txt" - releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() + def releaseJobNameFile = "${versionFolder}/releasejobname.txt" + def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents + def revision = readFile("${versionFolder}/revision.txt").trim() + def intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" return intrevision @@ -82,15 +84,16 @@ def appendcomponentinfo(rootpath, versionFolder, outputFolder) { // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - totalrevision = 0 + def totalrevision = 0 dir("${outputFolder}") { deleteDir() } for (artifactItem in artifactList) { (rootpath, versionFolder) = artifactItem echo "Inspecting folder: ${versionFolder}" - metaversionsignaturefile = "${versionFolder}/metaversion.txt" + def metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job + def intrevision = 0 if (fileExists("${metaversionsignaturefile}")) { intrevision = mergemeta(rootpath, versionFolder, outputFolder) } From 583cffb0969e1b58848d252b55403b7a47e784d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 20:35:46 +0000 Subject: [PATCH 069/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 6 ++---- versionnumbering/generate.groovy | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index b1a714f..a961dc8 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,6 +1,5 @@ env.CHECKSUMARTIFACTS = "checksums/**" -def CHECKSUMFOLDER = "${WORKSPACE}/checksums" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace @@ -8,6 +7,7 @@ def generatelist(checksummable) { if !fileExists("${SHA256SUM}") { error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } + def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ rm -rf "${CHECKSUMFOLDER}" mkdir -p "${CHECKSUMFOLDER}" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index a10fd76..2fdbb73 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,12 +1,9 @@ -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - env.PUBLISHABLEARTIFACTS = "publishable/**" -def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" -def PUBLISHABLEFILE = "${WORKSPACE}/publishable/list.json" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { + def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" sh """ rm -rf "${PUBLISHABLEFOLDER}" mkdir -p "${PUBLISHABLEFOLDER}" @@ -21,5 +18,6 @@ def generatelist(publishable) { } } def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) + def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) } \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 6f9f803..cb5e944 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,6 +1,6 @@ env.VERSIONARTIFACTS = "version/**" -def VERSIONFOLDER = "${WORKSPACE}/version" +VERSIONFOLDER = "${WORKSPACE}/version" def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { cs = autobuilder.commonscripts() From b7433f290fa8a05272a6973fd53b97b5896c584f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 21:29:07 +0000 Subject: [PATCH 070/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index a961dc8..e4c37ca 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,8 +4,20 @@ env.CHECKSUMARTIFACTS = "checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - if !fileExists("${SHA256SUM}") { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + def sha256sumfound = true + if (!fileExists("${SHA256SUM}")) { + def sha256sumfound = false + if (isUnix()) { + def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() + if (sha256sumpath) { + echo "Found sha256sum executable here: + env.SHA256SUM = sha256sumpath + sha256sumfound = true + } + } + if (!sha256sumfound) { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + } } def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ From 660003f7b49f43fb3f26a53255aab1382b2cbf42 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 21:50:35 +0000 Subject: [PATCH 071/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index e4c37ca..63c96eb 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -10,7 +10,7 @@ def generatelist(checksummable) { if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { - echo "Found sha256sum executable here: + echo "Found sha256sum executable here: ${sha256sumpath}" env.SHA256SUM = sha256sumpath sha256sumfound = true } From 53c2f1110619cf4d49ae0a65f91b4427744de9a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:20:30 +0000 Subject: [PATCH 072/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 63c96eb..0d76756 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -27,7 +27,7 @@ def generatelist(checksummable) { for (filepath in checksummable) { def files = findFiles(glob: filepath) for (file in files) { - if isUnix() { + if (isUnix()) { sh """#!/bin/bash "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ From 7c55b96a55782302e1fd92b3b10b0532a9d1b04b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:24:30 +0000 Subject: [PATCH 073/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 0d76756..e358b94 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -6,7 +6,7 @@ env.CHECKSUMARTIFACTS = "checksums/**" def generatelist(checksummable) { def sha256sumfound = true if (!fileExists("${SHA256SUM}")) { - def sha256sumfound = false + sha256sumfound = false if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { From dcdf2c5fd171c8c697d06d9dfedc854b33107ad9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:39:48 +0000 Subject: [PATCH 074/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index e358b94..a46cd43 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -25,16 +25,18 @@ def generatelist(checksummable) { mkdir -p "${CHECKSUMFOLDER}" """ for (filepath in checksummable) { + echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { + echo "Checksumming ${file.path}" if (isUnix()) { sh """#!/bin/bash - "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ } else { bat """ - "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ } } From e1e5c40a5470b9be9ada0959699c9718f77ffeb7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:55:21 +0000 Subject: [PATCH 075/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index a46cd43..8829198 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -43,3 +43,4 @@ def generatelist(checksummable) { } } +return this; From 419d491da023651aaa654db6c9e9f0e9fec1a5db Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 23:09:52 +0000 Subject: [PATCH 076/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 8829198..ef80fdd 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,9 +4,14 @@ env.CHECKSUMARTIFACTS = "checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - def sha256sumfound = true - if (!fileExists("${SHA256SUM}")) { - sha256sumfound = false + def sha256sumfound = false + if (env.SHA256SUM) { + if (fileExists("${SHA256SUM}")) { + sha256sumfound = true + } + } + if (!sha256sumfound) + { if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { @@ -15,9 +20,9 @@ def generatelist(checksummable) { sha256sumfound = true } } - if (!sha256sumfound) { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" - } + } + if (!sha256sumfound) { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ From 29d30dcc7817b2961e4ed97abd67507d73ea97f0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 23:20:01 +0000 Subject: [PATCH 077/195] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 1 - publishing/utils.groovy | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ef80fdd..ccf74db 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -33,7 +33,6 @@ def generatelist(checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { - echo "Checksumming ${file.path}" if (isUnix()) { sh """#!/bin/bash "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 2fdbb73..3fcb1f3 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -20,4 +20,6 @@ def generatelist(publishable) { def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) -} \ No newline at end of file +} + +return this; From 86c18871e6641998816aecbf3cd3461059f970f7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 09:48:30 +0000 Subject: [PATCH 078/195] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 3 ++- publishing/utils.groovy | 3 ++- versionnumbering/metaversion.groovy | 10 ---------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ccf74db..88496b8 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,5 +1,6 @@ -env.CHECKSUMARTIFACTS = "checksums/**" +def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 3fcb1f3..42ebced 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,5 +1,6 @@ -env.PUBLISHABLEARTIFACTS = "publishable/**" +def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index ff20a86..44bfbde 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -2,16 +2,6 @@ def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") -def currentContents(file, newline = "\n", defaultContents = "") { - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - def prefixline(rootpath, filepath) { def fileContents = readFile("${filepath}").trim() def fileContentsNormalized = fileContents.normalize() From 64023d515fc0460ba32cfa5cbb1fead0efa3015a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:24:08 +0000 Subject: [PATCH 079/195] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 meta/folder.groovy diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..d7c91ef --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,10 @@ + +METAFOLDERNAME = "meta" + +def reset() { + dir("${WORKSPACE}/${METAFOLDERNAME}") { + deleteDir() + } +} + +return this; \ No newline at end of file From d835cd4a1b02c781eaeee3751c97fc66988f85c9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:33:10 +0000 Subject: [PATCH 080/195] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index d7c91ef..2f4aaa3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,6 +2,7 @@ METAFOLDERNAME = "meta" def reset() { + echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } From 20697c05f1b45b686302ef82f2139d99dbe51d92 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:43:54 +0000 Subject: [PATCH 081/195] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 88496b8..02a11e0 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -25,7 +25,7 @@ def generatelist(checksummable) { if (!sha256sumfound) { error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } - def CHECKSUMFOLDER = "${WORKSPACE}/checksums" + def CHECKSUMFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/checksums" sh """ rm -rf "${CHECKSUMFOLDER}" mkdir -p "${CHECKSUMFOLDER}" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 42ebced..df7697c 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -4,7 +4,7 @@ env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { - def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" + def PUBLISHABLEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/publishable" sh """ rm -rf "${PUBLISHABLEFOLDER}" mkdir -p "${PUBLISHABLEFOLDER}" From 430fb3349644468bfc03a755894f57556171f627 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:46:49 +0000 Subject: [PATCH 082/195] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 02a11e0..5ec5ba9 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,5 +1,5 @@ -def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // SHA256SUM environment variable must be set to sha256sum utility diff --git a/publishing/utils.groovy b/publishing/utils.groovy index df7697c..72c6566 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,5 +1,5 @@ -def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties From a4d6817925d8094164e4c9f7eb53d5568c68a571 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:57:47 +0000 Subject: [PATCH 083/195] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2f4aaa3..ac82af4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,7 +1,7 @@ METAFOLDERNAME = "meta" -def reset() { +def metareset() { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() From 4d61150d49a920421d46aef3d661630eae2ddb47 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:09:47 +0000 Subject: [PATCH 084/195] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fd15492..89e2444 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -8,7 +8,7 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + stash allowEmpty: true, includes: 'autobuild/**', name: 'Clean workspace stash', useDefaultExcludes: false cleanWs() unstash name: 'Clean workspace stash' } @@ -17,21 +17,6 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { } } -// Start up boilderplate -def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autobuildroot}/Common_scripts") { - echo "Starting...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - - conditionalclean() - - lineendings = load("${commonscriptsroot}/platform/lineendings.groovy") - - def ch = load("${commonscriptsroot}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration() -} - def commonscripts() { return commonscriptsroot } @@ -40,6 +25,24 @@ def getautobuildpath() { return autobuildroot } +// Start up boilderplate +def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + def metafolder = load("${commonscripts()}/meta/folder.groovy") + metafolder.metareset() + + def lineendings = load("${commonscripts()}/platform/lineendings.groovy") + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration() +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } From ec457ffa08b034b2f6baa2db9a2661a57468574e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:11:49 +0000 Subject: [PATCH 085/195] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 89e2444..ef5ea7c 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot { +def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath From de74aae7682505e2c6f1ecbcbb8da7b4c6138cd6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:25:55 +0000 Subject: [PATCH 086/195] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 2 -- versionnumbering/generate.groovy | 12 ++++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ef5ea7c..bff3738 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -37,8 +37,6 @@ def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset() - def lineendings = load("${commonscripts()}/platform/lineendings.groovy") - def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cb5e944..935f026 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,9 +1,11 @@ -env.VERSIONARTIFACTS = "version/**" -VERSIONFOLDER = "${WORKSPACE}/version" +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") + +env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" +VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { - cs = autobuilder.commonscripts() + def cs = autobuilder.commonscripts() if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" @@ -17,7 +19,9 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - configurationhelper.saveConfigurationInVersionInfo(); + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.saveConfigurationInVersionInfo(); } return this; From a59b2c5a58fd6013aa50facb7551aa36cf88aab8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:45:45 +0000 Subject: [PATCH 087/195] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 3 +++ differences/do.groovy | 3 +++ 2 files changed, 6 insertions(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index bff3738..090c0f4 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -45,4 +45,7 @@ def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } +// Everyone wants to know this +autobuilder = this + return this; diff --git a/differences/do.groovy b/differences/do.groovy index b3039a9..7a550a9 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,4 +1,7 @@ +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" + def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() From 11d9fc7ee7386a495d88ac4f397967ed71dacb71 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:48:52 +0000 Subject: [PATCH 088/195] Create a meta folder seperate from other artifacts --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 935f026..f0f93be 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -20,7 +20,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - def ch = load("${commonscripts()}/configuration/configuration.groovy") + def ch = load("${cs}/configuration/configuration.groovy") ch.saveConfigurationInVersionInfo(); } From 387b799b47c8af15f9c8308b37fe5dfe4b79a797 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:16:23 +0000 Subject: [PATCH 089/195] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 41 +++---------------------- differences/do.groovy | 6 ++-- differences/show_release_differences.py | 20 ++++++------ meta/folder.groovy | 1 + 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 5ec5ba9..add3a50 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -2,50 +2,19 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" -// SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - def sha256sumfound = false - if (env.SHA256SUM) { - if (fileExists("${SHA256SUM}")) { - sha256sumfound = true - } - } - if (!sha256sumfound) - { - if (isUnix()) { - def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() - if (sha256sumpath) { - echo "Found sha256sum executable here: ${sha256sumpath}" - env.SHA256SUM = sha256sumpath - sha256sumfound = true - } - } - } - if (!sha256sumfound) { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" - } - def CHECKSUMFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/checksums" - sh """ - rm -rf "${CHECKSUMFOLDER}" - mkdir -p "${CHECKSUMFOLDER}" - """ + sums = "" for (filepath in checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { - if (isUnix()) { - sh """#!/bin/bash - "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" - """ - } - else { - bat """ - "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" - """ - } + def sum = sha1(file: file.path) + echo "Sum from file ${file.path} is ${sum}" + sums += "${sum}${nl}" } } + writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) } return this; diff --git a/differences/do.groovy b/differences/do.groovy index 7a550a9..8866400 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,17 +2,17 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEARTIFACTS}") { cs = autobuilder.commonscripts() if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 2d3ee0d..0813356 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,9 +29,9 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(versionfolder): - print("Creating folder (if not already there): {0}".format(versionfolder)) - os.makedirs(versionfolder, exist_ok = True) +def makeFolder(difffolder): + print("Creating folder (if not already there): {0}".format(difffolder)) + os.makedirs(difffolder, exist_ok = True) def printDivider(outfh, times = 1, length = 100): for _ in range(times): @@ -172,7 +172,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +183,14 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - versionfolder = args.versionfolder + difffolder = args.difffolder print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) - if versionfolder: - makeFolder(versionfolder) - outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + if difffolder: + makeFolder(difffolder) + outfh = open(os.path.join(difffolder, "differences.txt"), 'w') else: outfh = sys.stdout @@ -209,7 +209,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if versionfolder: + if difffolder: outfh.close() return 0 except Exception as e: diff --git a/meta/folder.groovy b/meta/folder.groovy index ac82af4..6b0b21c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ METAFOLDERNAME = "meta" +env.METAARTIFACTS = "${METAFOLDERNAME}/**" def metareset() { echo "Resetting meta folder" From 552370f9c1126e45b772762b274462ad26bd412b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:23:39 +0000 Subject: [PATCH 090/195] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index add3a50..ec3b10a 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,14 +4,14 @@ env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { + load("${autobuilder.commonscripts()}/platform/lineendings.groovy") sums = "" for (filepath in checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { def sum = sha1(file: file.path) - echo "Sum from file ${file.path} is ${sum}" - sums += "${sum}${nl}" + sums += "${sum} ${file.path}${nl}" // Format suitable for sha1sum utility } } writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) From 92e52dac5af83f9d04cec0f8abbe673cea445bf3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:27:40 +0000 Subject: [PATCH 091/195] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 1 + versionnumbering/generate.groovy | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ec3b10a..748472f 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,6 +1,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" +CHECKSUMFOLDER = "${metafolder.METAFOLDERNAME}/checksums" // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f0f93be..cdcf1c2 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,6 +1,5 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") - env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" From 0ad7692aed22ea93a9bd3e9b956c96e23cffbd27 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:36:24 +0000 Subject: [PATCH 092/195] Create a meta folder seperate from other artifacts --- differences/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 8866400..9e572af 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,10 +1,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" +DIFFERENCEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/differences" -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEARTIFACTS}") { +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEFOLDER}") { cs = autobuilder.commonscripts() - if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" From c04c82be175e05d9cbe7bc8f37f3d22538097a6b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 21:23:41 +0000 Subject: [PATCH 093/195] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 5 ++++- product/info.groovy | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 product/info.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 090c0f4..fc298ba 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -37,6 +37,9 @@ def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset() + def productinfo = load("${commonscripts()}/product/info.groovy") + productinfo.setname(productname, jirakey) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() } diff --git a/product/info.groovy b/product/info.groovy new file mode 100644 index 0000000..c41fc80 --- /dev/null +++ b/product/info.groovy @@ -0,0 +1,16 @@ + +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" +env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" + +def setname(productname = "", jirakey = "") { + def productdata = [ + 'name' : productname.trim(), + 'jirakey' : jirakey.trim() + ] + def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) + def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" + writeJSON(file: PRODUCTFILE, json: outJson, pretty: 4) +} + +return this; \ No newline at end of file From 0b491063dcd3de67fbe2c5e3543314ceb0d3f818 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 10:52:04 +0000 Subject: [PATCH 094/195] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 4 ++-- product/info.groovy | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fc298ba..ae3af20 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -38,7 +38,7 @@ def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commons metafolder.metareset() def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(productname, jirakey) + productinfo.setname(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/product/info.groovy b/product/info.groovy index c41fc80..05ed92a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -3,9 +3,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -def setname(productname = "", jirakey = "") { +def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { def productdata = [ 'name' : productname.trim(), + 'subcomponent' : subcomponentname.trim() 'jirakey' : jirakey.trim() ] def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) From f8bfaca3f08583a03dd077920637ef6b912b1a44 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 10:53:23 +0000 Subject: [PATCH 095/195] Create a meta folder seperate from other artifacts --- product/info.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 05ed92a..a908e7e 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -6,8 +6,8 @@ env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { def productdata = [ 'name' : productname.trim(), - 'subcomponent' : subcomponentname.trim() - 'jirakey' : jirakey.trim() + 'subcomponent' : subcomponentname.trim(), + 'jirakey' : jirakey.trim(), ] def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" From d1b409b2d49953164cd5f30524832285c8dfd521 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:32:14 +0000 Subject: [PATCH 096/195] Create a meta folder seperate from other artifacts --- product/info.groovy | 40 ++++++++++++--- versionnumbering/generate.groovy | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index a908e7e..71cf6fa 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,17 +1,43 @@ +load("${autobuilder.commonscripts()}/json/interface.groovy") + metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" +class ProductInfo extends JsonSerialised { + private String productname + private String subcomponentname + private String jirakey + + ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + productname = productName + subcomponentname = subcomponentName + jirakey = jiraKey + } + + def serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +theProductInfo = new ProductInfo() + def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - def productdata = [ - 'name' : productname.trim(), - 'subcomponent' : subcomponentname.trim(), - 'jirakey' : jirakey.trim(), - ] - def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) + theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) + def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - writeJSON(file: PRODUCTFILE, json: outJson, pretty: 4) + theProductInfo.saveJsonSerialised(PRODUCTFILE) } return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cdcf1c2..ad6214d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -3,6 +3,89 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +class GitInfo { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo +} + +class JenkinsBuildInfo { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate +} + +class BuildStampInfo { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep +} + +class ReleaseInfo { + private String releasejobname +} + +class VersionInfo { + private ComponentInfo componentinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + + private VersionInfo[] associatedversions +} + + def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { + + + version.generate("${SOURCE_REVISION}", "${SOURCE_PATH}", "${BASEBUILDNUMBER}", "${RELEASE_JOB_NAME}") + + VersionInfo() { + } + def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + + with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: + f.write(str(datetime.datetime.now())) + + with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: + f.write(releasejobname) + +} + def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From ae3ad5f31d3b6a57c261b18345069a9f0fa17520 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:34:36 +0000 Subject: [PATCH 097/195] Create a meta folder seperate from other artifacts --- json/interface.groovy | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 json/interface.groovy diff --git a/json/interface.groovy b/json/interface.groovy new file mode 100644 index 0000000..8962578 --- /dev/null +++ b/json/interface.groovy @@ -0,0 +1,7 @@ + +abstract class JsonSerialised { + def serialisationdata() + def saveJsonSerialised(String outputfile, int prettylevel = 4) +} + +return this; From bba686364164d7f8e781837ee06aa068e842d175 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:39:03 +0000 Subject: [PATCH 098/195] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 71cf6fa..979d0b5 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,5 +1,5 @@ -load("${autobuilder.commonscripts()}/json/interface.groovy") +def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" From f8f739e6fdd7b6efad937cc7ea60121541c02fdc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:22:46 +0000 Subject: [PATCH 099/195] Create a meta folder seperate from other artifacts --- json/interface.groovy | 4 ++-- product/info.groovy | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index 8962578..d238707 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,7 +1,7 @@ abstract class JsonSerialised { - def serialisationdata() - def saveJsonSerialised(String outputfile, int prettylevel = 4) + def serialisationdata(); + def saveJsonSerialised(String outputfile, int prettylevel = 4); } return this; diff --git a/product/info.groovy b/product/info.groovy index 979d0b5..9b2551c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -11,6 +11,7 @@ class ProductInfo extends JsonSerialised { private String jirakey ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + super() productname = productName subcomponentname = subcomponentName jirakey = jiraKey From babca972e743871e9e1178aafc6d4380a2520691 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:25:14 +0000 Subject: [PATCH 100/195] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 9b2551c..783b84c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo extends JsonSerialised { +class ProductInfo extends ji.JsonSerialised { private String productname private String subcomponentname private String jirakey From 5d05f9baeab64949a471f92cfc315dc72ee7028d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:26:48 +0000 Subject: [PATCH 101/195] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 783b84c..8b8d3e2 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo extends ji.JsonSerialised { +class ProductInfo { private String productname private String subcomponentname private String jirakey From 210e17c2e88acc28edc3ec3c2fa63da4c8ce77d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:29:20 +0000 Subject: [PATCH 102/195] Create a meta folder seperate from other artifacts --- json/interface.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index d238707..4cd2628 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,7 +1,7 @@ abstract class JsonSerialised { - def serialisationdata(); - def saveJsonSerialised(String outputfile, int prettylevel = 4); + abstract void serialisationdata(); + abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } return this; From 6ed97583368d97dd9badd659c8429a86b94de1e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:37:04 +0000 Subject: [PATCH 103/195] Create a meta folder seperate from other artifacts --- product/info.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 8b8d3e2..536d336 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -26,9 +26,9 @@ class ProductInfo { return data } - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + def saveJsonSerialised(script, String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) } } @@ -38,7 +38,7 @@ def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, ji theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(PRODUCTFILE) + theProductInfo.saveJsonSerialised(this, PRODUCTFILE) } return this; \ No newline at end of file From bf13a8f5a3b35f461e2fca45c5232055154f1aa5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 16:03:24 +0000 Subject: [PATCH 104/195] Create a meta folder seperate from other artifacts --- json/interface.groovy | 26 +++++++++++ versionnumbering/generate.groovy | 75 +------------------------------- 2 files changed, 28 insertions(+), 73 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index 4cd2628..fae1fe3 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -4,4 +4,30 @@ abstract class JsonSerialised { abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } +class GitInfo { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo +} + +class JenkinsBuildInfo { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate +} + +class BuildStampInfo { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep +} + +class ReleaseInfo { + private String releasejobname +} + return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ad6214d..3d498e9 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,34 +1,10 @@ +def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") + metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -class GitInfo { - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo -} - -class JenkinsBuildInfo { - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate -} - -class BuildStampInfo { - private String buildstampprefix - private int revision - private String buildstamp - private String buildstampcommasep -} - -class ReleaseInfo { - private String releasejobname -} - class VersionInfo { private ComponentInfo componentinfo private GitInfo gitinfo @@ -37,53 +13,6 @@ class VersionInfo { private ReleaseInfo releaseinfo private VersionInfo[] associatedversions -} - - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { - - - version.generate("${SOURCE_REVISION}", "${SOURCE_PATH}", "${BASEBUILDNUMBER}", "${RELEASE_JOB_NAME}") - - VersionInfo() { - } - def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - - with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: - f.write(releasejobname) - } def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { From efdf4ae8b854e64b0109205347cb03779a1c3ba1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 16:17:20 +0000 Subject: [PATCH 105/195] Create a meta folder seperate from other artifacts --- json/interface.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index fae1fe3..c9cd9f9 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -4,7 +4,7 @@ abstract class JsonSerialised { abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } -class GitInfo { +public class GitInfo { private String sourcepath private String sourcerevision private String gitrevision @@ -12,21 +12,21 @@ class GitInfo { private String remoteinfo } -class JenkinsBuildInfo { +public class JenkinsBuildInfo { private String buildjobname private int buildnumber private String buildmachine private String builddate } -class BuildStampInfo { +public class BuildStampInfo { private String buildstampprefix private int revision private String buildstamp private String buildstampcommasep } -class ReleaseInfo { +public class ReleaseInfo { private String releasejobname } From 7d0b8095a30fde7023129efd378b96a78957e83a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:30:30 +0000 Subject: [PATCH 106/195] Pass script object to custom objects --- autobuilder.groovy | 4 ++-- product/info.groovy | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ae3af20..ffaaf9f 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -38,7 +38,7 @@ def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jira metafolder.metareset() def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(productname, subcomponentname, jirakey) + productinfo.setname(script, productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/product/info.groovy b/product/info.groovy index 536d336..1e0af6d 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,12 +5,14 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo { +class ProductInfo implements Serializable { + private script private String productname private String subcomponentname private String jirakey - ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { + this.script = script super() productname = productName subcomponentname = subcomponentName @@ -26,7 +28,7 @@ class ProductInfo { return data } - def saveJsonSerialised(script, String outputfile, int prettylevel = 4) { + def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) } @@ -34,11 +36,11 @@ class ProductInfo { theProductInfo = new ProductInfo() -def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) +def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { + theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(this, PRODUCTFILE) + theProductInfo.saveJsonSerialised(PRODUCTFILE) } return this; \ No newline at end of file From 518b60a5e8f83136d3168b84c2ec993c467fd1fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:32:24 +0000 Subject: [PATCH 107/195] Pass script object to custom objects --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 1e0af6d..19f5df0 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -12,8 +12,8 @@ class ProductInfo implements Serializable { private String jirakey ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { - this.script = script super() + this.script = script productname = productName subcomponentname = subcomponentName jirakey = jiraKey From ce0643254bbbd99ef8bc4dbdca6eb34136a585c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:34:40 +0000 Subject: [PATCH 108/195] Pass script object to custom objects --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 19f5df0..78e281a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -34,7 +34,7 @@ class ProductInfo implements Serializable { } } -theProductInfo = new ProductInfo() +theProductInfo = null def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) From 05284a7607fbf8f3ce9b3b0c560047d576af942d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:52:07 +0000 Subject: [PATCH 109/195] Pass script object to custom objects --- versionnumbering/generate.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3d498e9..3d35e23 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -5,8 +5,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +def pi = load("${autobuilder.commonscripts()}/product/info.groovy") + class VersionInfo { - private ComponentInfo componentinfo + private ProductInfo productinfo private GitInfo gitinfo private JenkinsBuildInfo jenkinsbuildinfo private BuildStampInfo buildstampinfo From a1a36f47b7d91313a0d30a61aee7f98af34a1be8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 18:25:17 +0000 Subject: [PATCH 110/195] Pass script object to custom objects --- json/interface.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json/interface.groovy b/json/interface.groovy index c9cd9f9..58f9eb7 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,5 +1,5 @@ -abstract class JsonSerialised { +abstract class GISJsonSerialised { abstract void serialisationdata(); abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } From f4753020d9c0e901b0343ac4d7ff195ffc4a40c2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:18:27 +0000 Subject: [PATCH 111/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- autobuilder.groovy | 6 +- json/interface.groovy | 33 ------ meta/folder.groovy | 180 ++++++++++++++++++++++++++++++- product/info.groovy | 46 -------- versionnumbering/generate.groovy | 13 --- 5 files changed, 181 insertions(+), 97 deletions(-) delete mode 100644 json/interface.groovy delete mode 100644 product/info.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index ffaaf9f..dfc470b 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -35,10 +35,8 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() def metafolder = load("${commonscripts()}/meta/folder.groovy") - metafolder.metareset() - - def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(script, productname, subcomponentname, jirakey) + metafolder.metareset(script) + metafolder.setproductinfo(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/json/interface.groovy b/json/interface.groovy deleted file mode 100644 index 58f9eb7..0000000 --- a/json/interface.groovy +++ /dev/null @@ -1,33 +0,0 @@ - -abstract class GISJsonSerialised { - abstract void serialisationdata(); - abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); -} - -public class GitInfo { - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo -} - -public class JenkinsBuildInfo { - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate -} - -public class BuildStampInfo { - private String buildstampprefix - private int revision - private String buildstamp - private String buildstampcommasep -} - -public class ReleaseInfo { - private String releasejobname -} - -return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 6b0b21c..a8bc619 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,11 +2,189 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" -def metareset() { +class GISJsonSerialised implements Serializable { + protected script + + GISJsonSerialied(script) { + this.script = script + } + + abstract void serialisationdata(); + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + private String productname + private String subcomponentname + private String jirakey + + def set(script, productname, subcomponentname, jirakey) { + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jiraKey + } + + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + set(productname, subcomponentname, jirakey) + } + + def serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + super(script) + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + def serialisationdata() { + def data = [ + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + + JenkinsBuildInfo(script, buildjobname = "", buildnumber = 0, buildmachine = "", builddate = "") { + super(script) + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + def serialisationdata() { + def data = [ + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep + + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + super(script) + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + + def serialisationdata() { + def data = [ + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + 'buildstampcommasep' : buildstampcommasep, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + private String releasejobname + + BuildStampInfo(script, releasejobname = "") { + super(script) + this.releasejobname = releasejobname + } + + def serialisationdata() { + def data = [ + 'releasejobname' : releasejobname, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + + private MetaInfo[] associatedmeta + + MetaInfo(script) { + super(script) + productinfo(script) + gitinfo(script) + jenkinsbuildinfo(script) + buildstampinfo(script) + releaseinfo(script) + } + + def serialisationdata() { + def data = [ + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + ] + return datasetProductInfo + } + + def setproductinfo(productname, subcomponentname, jirakey) { + productinfo.set(productname, subcomponentname, jirakey) + } + + +MetaInfo theMetaData = new MetaInfo(this) + +def metareset(script) { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } + theMetaData = new MetaInfo(this) } +def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { + theMetaData.setproductinfo(productname, subcomponentname, jirakey) +} + + + return this; \ No newline at end of file diff --git a/product/info.groovy b/product/info.groovy deleted file mode 100644 index 78e281a..0000000 --- a/product/info.groovy +++ /dev/null @@ -1,46 +0,0 @@ - -def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") - -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" -env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" - -class ProductInfo implements Serializable { - private script - private String productname - private String subcomponentname - private String jirakey - - ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { - super() - this.script = script - productname = productName - subcomponentname = subcomponentName - jirakey = jiraKey - } - - def serialisationdata() { - def data = [ - 'productname' : productname, - 'subcomponentname' : subcomponentname, - 'jirakey' : jirakey, - ] - return data - } - - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) - } -} - -theProductInfo = null - -def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) - - def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(PRODUCTFILE) -} - -return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3d35e23..48f8bb6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,22 +1,9 @@ -def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") - metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" def pi = load("${autobuilder.commonscripts()}/product/info.groovy") -class VersionInfo { - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo - - private VersionInfo[] associatedversions -} - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From df5d11caa3f562bc5062d58aa50cf78b72a7784e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:37:27 +0000 Subject: [PATCH 112/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- versionnumbering/generate.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 48f8bb6..b576201 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -2,8 +2,6 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -def pi = load("${autobuilder.commonscripts()}/product/info.groovy") - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From 6323d8f5ebd5d1775da703bd93aadb78a42e31ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:45:39 +0000 Subject: [PATCH 113/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8bc619..0e800a4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -3,7 +3,7 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" class GISJsonSerialised implements Serializable { - protected script + def script GISJsonSerialied(script) { this.script = script From 397727d4f1bec0be03c5d7feaac10cea5f5a116a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:47:17 +0000 Subject: [PATCH 114/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0e800a4..ca4680f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -5,7 +5,7 @@ env.METAARTIFACTS = "${METAFOLDERNAME}/**" class GISJsonSerialised implements Serializable { def script - GISJsonSerialied(script) { + GISJsonSerialised(script) { this.script = script } From 746838240f0296d955740836debf602c820478a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:49:13 +0000 Subject: [PATCH 115/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ca4680f..65d4708 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -124,7 +124,7 @@ class BuildStampInfo extends GISJsonSerialised { class ReleaseInfo extends GISJsonSerialised { private String releasejobname - BuildStampInfo(script, releasejobname = "") { + ReleaseInfo(script, releasejobname = "") { super(script) this.releasejobname = releasejobname } From bba626b4175213c090a813b9560b12bce63630b6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:51:29 +0000 Subject: [PATCH 116/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 65d4708..46ae4f3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -169,7 +169,7 @@ class MetaInfo extends GISJsonSerialised { def setproductinfo(productname, subcomponentname, jirakey) { productinfo.set(productname, subcomponentname, jirakey) } - +} MetaInfo theMetaData = new MetaInfo(this) @@ -185,6 +185,4 @@ def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_N theMetaData.setproductinfo(productname, subcomponentname, jirakey) } - - return this; \ No newline at end of file From 326a38f0879ebbae0cd1e7743cabd659c8e5e09a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:03:51 +0000 Subject: [PATCH 117/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 46ae4f3..f13cdaa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,14 +2,14 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" -class GISJsonSerialised implements Serializable { +abstract class GISJsonSerialised implements Serializable { def script GISJsonSerialised(script) { this.script = script } - abstract void serialisationdata(); + abstract Map serialisationdata(); def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -33,7 +33,7 @@ class ProductInfo extends GISJsonSerialised { set(productname, subcomponentname, jirakey) } - def serialisationdata() { + Map serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, @@ -59,7 +59,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - def serialisationdata() { + Map serialisationdata() { def data = [ 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, @@ -85,7 +85,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - def serialisationdata() { + Map serialisationdata() { def data = [ 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, @@ -110,7 +110,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - def serialisationdata() { + Map serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, 'revision' : revision, @@ -129,7 +129,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - def serialisationdata() { + Map serialisationdata() { def data = [ 'releasejobname' : releasejobname, ] @@ -153,15 +153,25 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo(script) buildstampinfo(script) releaseinfo(script) + associatedmeta = [] } - def serialisationdata() { + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(serialisationdata(metaitem)) + } + return associated + } + + Map serialisationdata() { def data = [ 'productinfo' : productinfo.serialisationdata(), 'gitinfo' : gitinfo.serialisationdata(), 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), + 'associatedmeta' : associatedmetaserialised(), ] return datasetProductInfo } From c48b95b3f692ceb39d81c2409ba39eedf19dff64 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:38:55 +0000 Subject: [PATCH 118/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f13cdaa..afcd6f7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -148,11 +148,11 @@ class MetaInfo extends GISJsonSerialised { MetaInfo(script) { super(script) - productinfo(script) - gitinfo(script) - jenkinsbuildinfo(script) - buildstampinfo(script) - releaseinfo(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfoo = new JenkinsBuildInfo(script) + buildstampinfoo = new BuildStampInfo(script) + releaseinfoo = new ReleaseInfo(script) associatedmeta = [] } From b4dd72684c56a7f4d27d652f85e922c28a9b63ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:41:46 +0000 Subject: [PATCH 119/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index afcd6f7..ad3636d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -22,7 +22,7 @@ class ProductInfo extends GISJsonSerialised { private String subcomponentname private String jirakey - def set(script, productname, subcomponentname, jirakey) { + def set(productname, subcomponentname, jirakey) { this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jiraKey From 98311b6e4f2640da54ff0c021867261564e05a94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 19:38:29 +0000 Subject: [PATCH 120/195] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- autobuilder.groovy | 2 +- meta/folder.groovy | 62 ++++++++++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index dfc470b..fab812a 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -36,7 +36,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset(script) - metafolder.setproductinfo(productname, subcomponentname, jirakey) + metafolder.theMetaData.productinfo.set(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/meta/folder.groovy b/meta/folder.groovy index ad3636d..d4caa5d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -9,7 +9,7 @@ abstract class GISJsonSerialised implements Serializable { this.script = script } - abstract Map serialisationdata(); + abstract LinkedHashMap serialisationdata(); def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -22,18 +22,20 @@ class ProductInfo extends GISJsonSerialised { private String subcomponentname private String jirakey + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + def set(productname, subcomponentname, jirakey) { this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jiraKey + this.jirakey = jirakey } - ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { - super(script) - set(productname, subcomponentname, jirakey) - } - - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, @@ -59,7 +61,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, @@ -85,7 +87,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, @@ -110,7 +112,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, 'revision' : revision, @@ -129,7 +131,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'releasejobname' : releasejobname, ] @@ -138,34 +140,40 @@ class ReleaseInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo + private String format = '0.1' + ProductInfo productinfo + GitInfo gitinfo + JenkinsBuildInfo jenkinsbuildinfo + BuildStampInfo buildstampinfo + ReleaseInfo releaseinfo - private MetaInfo[] associatedmeta + private ArrayList associatedmeta MetaInfo(script) { super(script) productinfo = new ProductInfo(script) gitinfo = new GitInfo(script) - jenkinsbuildinfoo = new JenkinsBuildInfo(script) - buildstampinfoo = new BuildStampInfo(script) - releaseinfoo = new ReleaseInfo(script) - associatedmeta = [] + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + associatedmeta = new ArrayList() } List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { - associated.add(serialisationdata(metaitem)) + associated.add(metaitem.serialisationdata()) } return associated } - Map serialisationdata() { + void addAssociatedMetaInfo(mi) { + associatedmeta.add(mi) + } + + LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'productinfo' : productinfo.serialisationdata(), 'gitinfo' : gitinfo.serialisationdata(), 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), @@ -173,7 +181,7 @@ class MetaInfo extends GISJsonSerialised { 'releaseinfo' : releaseinfo.serialisationdata(), 'associatedmeta' : associatedmetaserialised(), ] - return datasetProductInfo + return data } def setproductinfo(productname, subcomponentname, jirakey) { @@ -191,8 +199,4 @@ def metareset(script) { theMetaData = new MetaInfo(this) } -def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theMetaData.setproductinfo(productname, subcomponentname, jirakey) -} - return this; \ No newline at end of file From 59d37c79a58e01bef069a8e8f395677395aa8105 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 20:25:33 +0000 Subject: [PATCH 121/195] Start using meta info for storing configuration --- autobuilder.groovy | 13 ++++---- configuration/configuration.groovy | 33 ++++---------------- meta/folder.groovy | 48 ++++++++++++++++++++++++++---- versionnumbering/generate.groovy | 3 -- 4 files changed, 57 insertions(+), 40 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fab812a..a82e891 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolder = load("${commonscripts()}/meta/folder.groovy") + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -26,7 +28,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -34,12 +36,13 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - metafolder.metareset(script) - metafolder.theMetaData.productinfo.set(productname, subcomponentname, jirakey) + def meta = metafolder.metareset(script) + meta.productinfo.set(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration() + ch.registerPipelineScriptConfiguration(meta) + + return meta } def configurationhelper() { diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 1f0c095..30b7086 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,37 +1,16 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -configurationCollection = [:] - -def registerConfigurationInfo(String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - def ConfigurationInfo = [configurationURL, configurationVersion, configurationSHA] - configurationCollection.put(configurationName, ConfigurationInfo) +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) } -def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) +def registerPipelineScriptConfiguration(meta) { + registerConfigurationInfo(meta, 'pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } -def saveConfigurationInVersionInfo() { - dir("${WORKSPACE}/version") { - echo "Writing configuration info to ${WORKSPACE}/version/configuration.txt" - writeFile file: 'configuration.txt', text: getFormattedConfiguration() - } -} - -String getFormattedConfiguration() { - String formattedInfo = "" - def keys = configurationCollection.keySet().sort() - for (key in keys) { - (configurationURL, configurationVersion, configurationSHA) = configurationCollection.get(key) - formattedLine = String.format("%s:%s,%s,%s%n", key, configurationURL, configurationVersion, configurationSHA) - formattedInfo += formattedLine - } - return formattedInfo -} - -def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(tool, gitURL, revision, gh.getGitInfo(targetDir)) + registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this diff --git a/meta/folder.groovy b/meta/folder.groovy index d4caa5d..2963e58 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -139,6 +139,31 @@ class ReleaseInfo extends GISJsonSerialised { } } +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String format = '0.1' ProductInfo productinfo @@ -146,7 +171,7 @@ class MetaInfo extends GISJsonSerialised { JenkinsBuildInfo jenkinsbuildinfo BuildStampInfo buildstampinfo ReleaseInfo releaseinfo - + private ArrayList configurationinfo private ArrayList associatedmeta MetaInfo(script) { @@ -156,9 +181,23 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo = new JenkinsBuildInfo(script) buildstampinfo = new BuildStampInfo(script) releaseinfo = new ReleaseInfo(script) + configurationinfo = new ArrayList() associatedmeta = new ArrayList() } + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return associated + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + configurationinfo.add(ci) + } + List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { @@ -179,6 +218,7 @@ class MetaInfo extends GISJsonSerialised { 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), 'associatedmeta' : associatedmetaserialised(), ] return data @@ -189,14 +229,12 @@ class MetaInfo extends GISJsonSerialised { } } -MetaInfo theMetaData = new MetaInfo(this) - -def metareset(script) { +MetaInfo metareset(script) { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } - theMetaData = new MetaInfo(this) + return new MetaInfo(script) } return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index b576201..b99467e 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -17,9 +17,6 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - - def ch = load("${cs}/configuration/configuration.groovy") - ch.saveConfigurationInVersionInfo(); } return this; From 35ae72bbdbbf7c34bdf47921be41b6b7522c7dd2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 20:29:56 +0000 Subject: [PATCH 122/195] Start using meta info for storing configuration --- autobuilder.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index a82e891..e7943c6 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,8 +4,6 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" -metafolder = load("${commonscripts()}/meta/folder.groovy") - def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -28,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -36,6 +34,7 @@ MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONE conditionalclean() + def metafolder = load("${commonscripts()}/meta/folder.groovy") def meta = metafolder.metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) From d9b91dee691a29ae78466b8cbc2fd528d9d9a745 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 22:52:20 +0000 Subject: [PATCH 123/195] Move functionality to meta --- autobuilder.groovy | 5 ++++- meta/folder.groovy | 13 ++++++++++++- versionnumbering/generate.groovy | 32 +++++++++++++++++++++++++++++--- versionnumbering/versioninfo.py | 25 +++---------------------- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index e7943c6..3a2a408 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -37,7 +37,10 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def metafolder = load("${commonscripts()}/meta/folder.groovy") def meta = metafolder.metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) - + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2963e58..6ae3d3d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -79,7 +79,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { private String buildmachine private String builddate - JenkinsBuildInfo(script, buildjobname = "", buildnumber = 0, buildmachine = "", builddate = "") { + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { super(script) this.buildjobname = buildjobname this.buildnumber = buildnumber @@ -87,6 +87,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } + def set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, @@ -131,6 +138,10 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } + def set(releasejobname) { + this.releasejobname = releasejobname + } + LinkedHashMap serialisationdata() { def data = [ 'releasejobname' : releasejobname, diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index b99467e..0019b24 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -2,16 +2,16 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { +def generate(meta, sourcerevision, sourcepath, basebuildnumber = "1000", versionfolder = "${VERSIONFOLDER}") { def cs = autobuilder.commonscripts() if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" """ } env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() @@ -20,3 +20,29 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } return this; + + +/* + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) +*/ diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 370d9fc..0fa097a 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -24,7 +24,6 @@ __date__ = '2017-11-29' __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -slaveMachine = os.getenv("NODE_NAME", default="-") MAXVERSIONNUMBER = 65535 def resetFolder(versionfolder): @@ -138,10 +137,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - +def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: f.write(versionprefix) @@ -157,9 +153,6 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: f.write(remoteurl) - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: f.write(buildstamp) @@ -167,15 +160,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: f.write(buildstampcommasep) - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: f.write(str(datetime.datetime.now())) - with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: - f.write(releasejobname) - def main(argv=None): # IGNORE:C0111 '''Command line options.''' @@ -207,9 +194,6 @@ USAGE parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") - parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") - parser.add_argument("-r", "--releasejobname", dest="releasejobname", action="store", default="-", help="release-to-git build job name [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -219,12 +203,9 @@ USAGE versionfolder = args.versionfolder gitrootfolder = args.gitrootfolder baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber - releasejobname = args.releasejobname print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber, "Release job name:", releasejobname) + print("Base version number:", baseversionnumber) resetFolder(versionfolder) @@ -235,7 +216,7 @@ USAGE buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname) + writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) return 0 except Exception as e: From 84c105946be4abdcb35ad85da8b847e81875a511 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:08:11 +0000 Subject: [PATCH 124/195] Improve meta gathering --- meta/folder.groovy | 15 ++++ versionnumbering/generate.groovy | 134 +++++++++++++++++++++++-------- versionnumbering/versioninfo.py | 53 ------------ 3 files changed, 114 insertions(+), 88 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6ae3d3d..8b638a9 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -61,6 +61,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + def set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, @@ -119,6 +127,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + def set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 0019b24..f9683e6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,48 +1,112 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +ddef getGitRevision(String gitrootpath, int basebuildnumber) { + dir( gitrootpath) { + def int MAXVERSIONNUMBER = 65535 -def generate(meta, sourcerevision, sourcepath, basebuildnumber = "1000", versionfolder = "${VERSIONFOLDER}") { - def cs = autobuilder.commonscripts() - if (isUnix()) { - sh """ - "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" - """ + def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count") + echo "Change count: ${changecount}" + def int adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + echo "Adjusted change count: ${adjustedchangecount}" + + def gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD") + echo "Revision: ${gitrevision}" + + def remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin") + echo "Remote info: ${remoteinfo}" + + def remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin") + echo "Remote URL: ${remoteurl}" } - else { - bat """ - "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" - """ - } - env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() - env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() - env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] } -return this; +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch) { + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + def defaultversionprefix = "1.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } -/* - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) + def mastertypeversionprefix = "1.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) + echo "sourcebranch is now: ${sourcebranch}" + def splits2 = sourcebranch.split('\\.') + echo "Splits2 is: ${splits2}" + if (splits2.size() < 1) { + def thirdpartyversionprefix = "1.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + splits = [] + for (split in splits2) { + echo "Split is: ${split}" + if (split.isInteger()) { + splits.add(split) + } + else if (split == "x") { + return mastertypeversionprefix + } + } + echo "Splits is: ${splits}" - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) + def versionprefix = "" + if (splits.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int splitnumbercount = 0 + for (split in splits) { + echo "Split is ${split}" + if (split.isInteger()) { + versionprefix += split + "." + splitnumbercount += 1 + } + else if (splitnumbercount > 2) { + break + } + } + if (splitnumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + if (splitnumbercount == 1) { + versionprefix += "0.0." + } + else if (splitnumbercount == 2) { + versionprefix += "0." + } + } + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) + def versionprefix = getVersionPrefix(sourcerevision) - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) -*/ + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + + return buildstamp +} + +return this; \ No newline at end of file diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 0fa097a..3d2a80d 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -23,57 +23,6 @@ __version__ = 0.1 __date__ = '2017-11-29' __updated__ = '2017-11-29' -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -MAXVERSIONNUMBER = 65535 - -def resetFolder(versionfolder): - print("Removing folders and contents of: {0}".format(versionfolder)) - try: - shutil.rmtree(versionfolder) - except FileNotFoundError: - pass - - print("Creating folder: {0}".format(versionfolder)) - os.makedirs(versionfolder) - -def getGitRevision(gitrootpath, basenumber): - cwd = os.getcwd() - - path = os.path.expanduser(gitrootpath) - print("Changing current working directory to: {0}".format(path)) - os.chdir(path) - - print("Determining revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - changecount = completed.stdout.decode("utf-8").strip() - print("Change count: {0}".format(changecount)) - adjustedchangecount = int((int(changecount) + int(basenumber)) % (MAXVERSIONNUMBER + 1)) - print("Adjusted change count: {0}".format(adjustedchangecount)) - - print("Determining current revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - revision = completed.stdout.decode("utf-8").strip() - print("Revision: {0}".format(revision)) - - print("Determining remote info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteinfo = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteinfo)) - - print("Determining remote url using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "get-url", "--push", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteurl = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteurl)) - - os.chdir(cwd) - print("Restored current working directory to: {0}".format(cwd)) - - return (adjustedchangecount, revision, remoteinfo, remoteurl) - # Handle: # origin/master # origin/2.0.x @@ -207,8 +156,6 @@ USAGE print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) print("Base version number:", baseversionnumber) - resetFolder(versionfolder) - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) versionprefix = getVersionPrefix(sourcebranch) From 654b4fa87be1ba9de08f3aefcd5793740f2391bd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:08:43 +0000 Subject: [PATCH 125/195] Improve meta gathering --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f9683e6..69bdd4d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,5 +1,5 @@ -ddef getGitRevision(String gitrootpath, int basebuildnumber) { +def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 From 963dabcc6e61dd6a97e76f5243be4786789d1ca9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:13:47 +0000 Subject: [PATCH 126/195] Improve meta gathering --- meta/folder.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8b638a9..690ffaa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -29,7 +29,7 @@ class ProductInfo extends GISJsonSerialised { this.jirakey = jirakey } - def set(productname, subcomponentname, jirakey) { + void set(String productname, String subcomponentname, String jirakey) { this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jirakey @@ -61,7 +61,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - def set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) { + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -95,7 +95,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - def set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine @@ -127,7 +127,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - def set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp @@ -153,7 +153,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - def set(releasejobname) { + void set(String releasejobname) { this.releasejobname = releasejobname } @@ -220,7 +220,7 @@ class MetaInfo extends GISJsonSerialised { } void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { - ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) configurationinfo.add(ci) } @@ -263,4 +263,4 @@ MetaInfo metareset(script) { return new MetaInfo(script) } -return this; \ No newline at end of file +return this; From 0fe07ee41231719f3b8a42b7fd089ac590a09602 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:22:00 +0000 Subject: [PATCH 127/195] Improve meta gathering --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 690ffaa..045d67d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -216,7 +216,7 @@ class MetaInfo extends GISJsonSerialised { for (configurationitem in configurationinfo) { configuration.add(configurationitem.serialisationdata()) } - return associated + return configuration } void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 69bdd4d..3c475bc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,20 +1,24 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count") + def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count").trim() echo "Change count: ${changecount}" - def int adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) echo "Adjusted change count: ${adjustedchangecount}" - def gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD") + gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD").trim() echo "Revision: ${gitrevision}" - def remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin") + remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin").trim() echo "Remote info: ${remoteinfo}" - def remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin") + remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] @@ -41,9 +45,7 @@ def getVersionPrefix(String sourcebranch) { return mastertypeversionprefix } - echo "sourcebranch is now: ${sourcebranch}" def splits2 = sourcebranch.split('\\.') - echo "Splits2 is: ${splits2}" if (splits2.size() < 1) { def thirdpartyversionprefix = "1.25000.0." echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" @@ -51,7 +53,6 @@ def getVersionPrefix(String sourcebranch) { } splits = [] for (split in splits2) { - echo "Split is: ${split}" if (split.isInteger()) { splits.add(split) } @@ -59,7 +60,6 @@ def getVersionPrefix(String sourcebranch) { return mastertypeversionprefix } } - echo "Splits is: ${splits}" def versionprefix = "" if (splits.size() < 1) { @@ -69,7 +69,6 @@ def getVersionPrefix(String sourcebranch) { else { def int splitnumbercount = 0 for (split in splits) { - echo "Split is ${split}" if (split.isInteger()) { versionprefix += split + "." splitnumbercount += 1 @@ -96,6 +95,11 @@ def getVersionPrefix(String sourcebranch) { } String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) def versionprefix = getVersionPrefix(sourcerevision) From fa26214f9e65cde90568843a5cf683cb935252d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:26:40 +0000 Subject: [PATCH 128/195] Improve meta gathering --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 3a2a408..50a1655 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -39,7 +39,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 838460a826e7ad122a507e641260b736e7149b0b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:30:56 +0000 Subject: [PATCH 129/195] Improve meta gathering --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 50a1655..bbe7bb9 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -39,7 +39,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now.toString()) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 6ece33765635a185bd017e2eb1e18797379bbe13 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:40:37 +0000 Subject: [PATCH 130/195] Improve meta gathering --- versionnumbering/generate.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3c475bc..16fb1a9 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -7,18 +7,18 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count").trim() + def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) echo "Adjusted change count: ${adjustedchangecount}" - gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD").trim() + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() echo "Revision: ${gitrevision}" - remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin").trim() + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 9e512c2774e23037348c63eca63122313409d110 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 11:32:04 +0000 Subject: [PATCH 131/195] Improve meta gathering --- differences/do.groovy | 10 ++++------ meta/folder.groovy | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 9e572af..35e0980 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,10 +1,8 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" -DIFFERENCEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/differences" - -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEFOLDER}") { - cs = autobuilder.commonscripts() +def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { + def String cs = autobuilder.commonscripts() + def String difffolder = meta.folder() + def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" diff --git a/meta/folder.groovy b/meta/folder.groovy index 045d67d..25d5f1f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,7 +1,4 @@ -METAFOLDERNAME = "meta" -env.METAARTIFACTS = "${METAFOLDERNAME}/**" - abstract class GISJsonSerialised implements Serializable { def script @@ -20,7 +17,7 @@ abstract class GISJsonSerialised implements Serializable { class ProductInfo extends GISJsonSerialised { private String productname private String subcomponentname - private String jirakey + String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { super(script) @@ -191,6 +188,10 @@ class ConfigurationInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { + private String METAFOLDERNAME = "meta" + private String METAFILENAME = "info.json" + private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private String format = '0.1' ProductInfo productinfo GitInfo gitinfo @@ -250,17 +251,26 @@ class MetaInfo extends GISJsonSerialised { return data } - def setproductinfo(productname, subcomponentname, jirakey) { - productinfo.set(productname, subcomponentname, jirakey) + + String folder() { + return METAFOLDERNAME + } + + String generateartifacts() { + def String metafilepath = "${METAFOLDERNAME}/${METAFILENAME}" + saveJsonSerialised(metafilepath) + return METAARTIFACTS } } MetaInfo metareset(script) { echo "Resetting meta folder" - dir("${WORKSPACE}/${METAFOLDERNAME}") { + def mi = new MetaInfo(script) + + dir(mi.folder()) { deleteDir() } - return new MetaInfo(script) + return mi } return this; From bebea3f7f7fd9d520ba415cadb6422bee24d6f5a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 13:15:10 +0000 Subject: [PATCH 132/195] Add checksum info to meta --- checksum/utils.groovy | 15 +++----- meta/folder.groovy | 83 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 748472f..c090bf3 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,21 +1,14 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" -CHECKSUMFOLDER = "${metafolder.METAFOLDERNAME}/checksums" - // Generate a checksum file of from a list of files/directories relative to workspace -def generatelist(checksummable) { - load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - sums = "" +def generatelist(meta, checksummable) { for (filepath in checksummable) { - echo "Checksumming ${filepath}" + def checksumset = meta.checksuminfo.addchecksumset(filepath) def files = findFiles(glob: filepath) for (file in files) { - def sum = sha1(file: file.path) - sums += "${sum} ${file.path}${nl}" // Format suitable for sha1sum utility + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) } } - writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 25d5f1f..152121c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -187,6 +187,86 @@ class ConfigurationInfo extends GISJsonSerialised { } } +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath, String sum) { + super(checksumset.script) + this.filepath = filepath + this.sum = sum + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String artifactpath + private ArrayList checksumpairs + + ChecksumInfo(script, artifactpath) { + super(script) + this.artifactpath = artifactpath + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } + + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" @@ -198,6 +278,7 @@ class MetaInfo extends GISJsonSerialised { JenkinsBuildInfo jenkinsbuildinfo BuildStampInfo buildstampinfo ReleaseInfo releaseinfo + ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList associatedmeta @@ -208,6 +289,7 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo = new JenkinsBuildInfo(script) buildstampinfo = new BuildStampInfo(script) releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() associatedmeta = new ArrayList() } @@ -246,6 +328,7 @@ class MetaInfo extends GISJsonSerialised { 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), 'associatedmeta' : associatedmetaserialised(), ] return data From 2b6378e8e41fe7ea26ead77643be8db2b5e8a761 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 18:33:55 +0000 Subject: [PATCH 133/195] Add checksum info to meta --- meta/folder.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 152121c..9a89b70 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -192,7 +192,7 @@ class ChecksumPair extends GISJsonSerialised { private String sum ChecksumPair(script, String filepath, String sum) { - super(checksumset.script) + super(script) this.filepath = filepath this.sum = sum } @@ -210,9 +210,10 @@ class ChecksumSet extends GISJsonSerialised { private String artifactpath private ArrayList checksumpairs - ChecksumInfo(script, artifactpath) { + ChecksumSet(script, artifactpath) { super(script) this.artifactpath = artifactpath + checksumpairs = new ArrayList() } ChecksumPair addChecksumPair(String filepath, String sum) { @@ -236,17 +237,20 @@ class ChecksumSet extends GISJsonSerialised { ] return data } - +} class ChecksumInfo extends GISJsonSerialised { private ArrayList checksumsets ChecksumInfo(script) { super(script) + checksumsets = new ArrayList() } ChecksumSet addChecksumSet(String artifactpath) { + script.echo "Entering addChecksumSet" def cs = new ChecksumSet(script, artifactpath) + script.echo "Created ChecksumSet" checksumsets.add(cs) return cs } @@ -339,8 +343,7 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts() { - def String metafilepath = "${METAFOLDERNAME}/${METAFILENAME}" + String generateartifacts(String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}") { saveJsonSerialised(metafilepath) return METAARTIFACTS } From 084e8367d7b1907771e65a9f1689686ca605fdc5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 18:40:12 +0000 Subject: [PATCH 134/195] Add checksum info to meta --- checksum/utils.groovy | 4 ++-- meta/folder.groovy | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index c090bf3..c2503cd 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -2,7 +2,7 @@ // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(meta, checksummable) { for (filepath in checksummable) { - def checksumset = meta.checksuminfo.addchecksumset(filepath) + def checksumset = meta.checksuminfo.addChecksumSet(filepath) def files = findFiles(glob: filepath) for (file in files) { def sum = sha1(file: file.path).trim() @@ -11,4 +11,4 @@ def generatelist(meta, checksummable) { } } -return this; +return this; \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy index 9a89b70..421fec8 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -248,9 +248,7 @@ class ChecksumInfo extends GISJsonSerialised { } ChecksumSet addChecksumSet(String artifactpath) { - script.echo "Entering addChecksumSet" def cs = new ChecksumSet(script, artifactpath) - script.echo "Created ChecksumSet" checksumsets.add(cs) return cs } From 8d90a54ebd266a26ed229d56bd5c741e054ad55e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 20:19:08 +0000 Subject: [PATCH 135/195] Add publishable items to meta --- meta/folder.groovy | 32 ++++++++++++++++++++++++++++++++ publishing/utils.groovy | 20 ++------------------ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 421fec8..7471f74 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -269,6 +269,22 @@ class ChecksumInfo extends GISJsonSerialised { } } +class PublishableInfo extends GISJsonSerialised { + private String filepath + + PublishableInfo(script, String filepath = "") { + super(script) + this.filepath = filepath + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" @@ -282,6 +298,7 @@ class MetaInfo extends GISJsonSerialised { ReleaseInfo releaseinfo ChecksumInfo checksuminfo private ArrayList configurationinfo + private ArrayList publishableinfo private ArrayList associatedmeta MetaInfo(script) { @@ -293,6 +310,7 @@ class MetaInfo extends GISJsonSerialised { releaseinfo = new ReleaseInfo(script) checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() + publishableinfo = new ArrayList() associatedmeta = new ArrayList() } @@ -304,11 +322,24 @@ class MetaInfo extends GISJsonSerialised { return configuration } + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + publishableinfo.add(pi) + } + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) configurationinfo.add(ci) } + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { @@ -331,6 +362,7 @@ class MetaInfo extends GISJsonSerialised { 'releaseinfo' : releaseinfo.serialisationdata(), 'configuration' : configurationinfoserialised(), 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), 'associatedmeta' : associatedmetaserialised(), ] return data diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 72c6566..145a7e9 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,26 +1,10 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" - // Generate a list of artifact files that are publishable to outside parties -def generatelist(publishable) { - def PUBLISHABLEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/publishable" - sh """ - rm -rf "${PUBLISHABLEFOLDER}" - mkdir -p "${PUBLISHABLEFOLDER}" - """ +def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { - def files = findFiles(glob: filepath) - for (file in files) { - if (!file.directory) { - filelist.add(file.path) - } - } + meta.addPublishable(filepath) } - def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) - def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" - writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) } return this; From a06e33d10d6690247fa569a7331eb376ebfb4ab3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 22:17:38 +0000 Subject: [PATCH 136/195] Add release information items to meta --- meta/folder.groovy | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7471f74..b5cc379 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -143,20 +143,35 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private String releasejobname + private String releaserepoURL + private ArrayList releaseitems - ReleaseInfo(script, releasejobname = "") { + ReleaseInfo(script, releaserepoURL = "") { super(script) - this.releasejobname = releasejobname + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() } - void set(String releasejobname) { - this.releasejobname = releasejobname + void set(String releaserepoURL = "") { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + releaseitems.add(releaseitem) + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items } LinkedHashMap serialisationdata() { def data = [ - 'releasejobname' : releasejobname, + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), ] return data } @@ -373,9 +388,21 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}") { + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), + ArrayList releasedfromsubmeta = new ArrayList(), + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = METAARTIFACTS + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } saveJsonSerialised(metafilepath) - return METAARTIFACTS + return artifactlist } } From f14ce9fc0cf3d85cbc66a8f4155515c538dd603c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 17:23:17 +0000 Subject: [PATCH 137/195] Deserialise meta from file and associate with job meta --- import/importer.groovy | 14 ++++ meta/folder.groovy | 154 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 import/importer.groovy diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..ba912f4 --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,14 @@ + +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: buildselector + ], + target: destpath + ) + meta.importrelated(destpath) +} \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy index b5cc379..9d90cee 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -6,7 +6,9 @@ abstract class GISJsonSerialised implements Serializable { this.script = script } - abstract LinkedHashMap serialisationdata(); + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -32,6 +34,12 @@ class ProductInfo extends GISJsonSerialised { this.jirakey = jirakey } + void deserialiseJson(rawmeta) { + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + jirakey = rawmeta['jirakey'] + } + LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, @@ -66,6 +74,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + void deserialiseJson(rawmeta) { + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, @@ -99,6 +115,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } + void deserialiseJson(rawmeta) { + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + } + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, @@ -131,6 +154,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + void deserialiseJson(rawmeta) { + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + buildstampcommasep = rawmeta['buildstampcommasep'] + } + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, @@ -168,6 +198,14 @@ class ReleaseInfo extends GISJsonSerialised { return items } + void deserialiseJson(rawmeta) { + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + } + LinkedHashMap serialisationdata() { def data = [ 'releaserepoURL' : releaserepoURL, @@ -191,6 +229,13 @@ class ConfigurationInfo extends GISJsonSerialised { this.configurationSHA = configurationSHA } + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + LinkedHashMap serialisationdata() { def data = [ 'configurationname' : configurationname, @@ -212,6 +257,11 @@ class ChecksumPair extends GISJsonSerialised { this.sum = sum } + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + LinkedHashMap serialisationdata() { def data = [ 'filepath' : filepath, @@ -245,6 +295,16 @@ class ChecksumSet extends GISJsonSerialised { return associated } + void deserialiseJson(rawmeta) { + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + LinkedHashMap serialisationdata() { def data = [ 'artifactpath' : artifactpath, @@ -276,6 +336,15 @@ class ChecksumInfo extends GISJsonSerialised { return associated } + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + LinkedHashMap serialisationdata() { def data = [ 'checksumsets' : checksumsetsserialised(), @@ -292,6 +361,10 @@ class PublishableInfo extends GISJsonSerialised { this.filepath = filepath } + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + } + LinkedHashMap serialisationdata() { def data = [ 'filepath' : filepath, @@ -301,11 +374,37 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { + private String FORMAT = "0.1" private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" private String METAARTIFACTS = "${METAFOLDERNAME}/**" - private String format = '0.1' + class RelatedMetaInfo extends GISJsonSerialised { + String offsetpath + MetaInfo meta + + RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + super(script) + this.offsetpath = offsetpath + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'offsetpath' : offsetpath, + 'meta' : meta.serialisationdata(), + ] + return data + } + } + + private String format = FORMAT ProductInfo productinfo GitInfo gitinfo JenkinsBuildInfo jenkinsbuildinfo @@ -314,7 +413,7 @@ class MetaInfo extends GISJsonSerialised { ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo - private ArrayList associatedmeta + private ArrayList associatedmeta MetaInfo(script) { super(script) @@ -326,7 +425,7 @@ class MetaInfo extends GISJsonSerialised { checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() publishableinfo = new ArrayList() - associatedmeta = new ArrayList() + associatedmeta = new ArrayList() } List configurationinfoserialised() { @@ -363,8 +462,51 @@ class MetaInfo extends GISJsonSerialised { return associated } - void addAssociatedMetaInfo(mi) { - associatedmeta.add(mi) + void deserialiseJson(rawmeta) { + if (rawmeta['format'] != FORMAT) { + script.error("Unsupported meta format found at ${filepath}") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + for (rawrelatedmeta in rawmeta['associatedmeta']) { + def rm = new RelatedMetaInfo(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + + def deserialiseJson(String filepath) { + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + MetaInfo importrelated(String destpath) { + MetaInfo mi = new MetaInfo(script) + + def importedmetafile = destpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + } + + RelatedMetaInfo rmi = new RelatedMetaInfo(script, destpath, mi) + associatedmeta.add(rmi) + + return mi } LinkedHashMap serialisationdata() { From 53878daf609b8eacf1dd815b89cc4d1e8507d5a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 18:51:51 +0000 Subject: [PATCH 138/195] Deserialise meta from file and associate with job meta --- import/importer.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index ba912f4..d3c97e4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -11,4 +11,6 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) meta.importrelated(destpath) -} \ No newline at end of file +} + +return this; \ No newline at end of file From 929c18f5b690313da022d54bd815b8b0aab76dc8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 19:54:32 +0000 Subject: [PATCH 139/195] Store offset paths relative to WORKSPACE if given as absolute paths --- meta/folder.groovy | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9d90cee..28fe85f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,3 +1,5 @@ +import java.nio.file.Paths +import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { def script @@ -494,16 +496,27 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } + + String relativize(String path) { + def Path thispath = Paths.get(path) + if (thispath.startsWith(script.WORKSPACE)) { + def Path wspath = Paths.get(script.WORKSPACE) + def Path wsrelativePath = wspath.relativize(thispath) + return wsrelativePath.toString() + } + return path + } MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) - def importedmetafile = destpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + METAFOLDERNAME + "/" + METAFILENAME if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, destpath, mi) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) return mi From 8d119dcdbc340866d01f763a592649a445d4cd38 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 20:06:10 +0000 Subject: [PATCH 140/195] Only show SHA code of configuration --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 031094b..cb9354b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -2,7 +2,7 @@ clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { - return clh.getCommandOutput('git log -1 --format=oneline') + return clh.getCommandOutput('git rev-parse HEAD') } } From 9b51bd7cf89adcbd1de56613d51d22336bec760c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 21:05:06 +0000 Subject: [PATCH 141/195] Handle more version number strings --- versionnumbering/generate.groovy | 46 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 16fb1a9..59fe2a3 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -28,6 +28,8 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { + def int MAXVERSIONNUMBER = 65535 + echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -51,43 +53,45 @@ def getVersionPrefix(String sourcebranch) { echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" return thirdpartyversionprefix } - splits = [] + for (split in splits2) { - if (split.isInteger()) { - splits.add(split) - } - else if (split == "x") { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" return mastertypeversionprefix } } + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + int integer = integerstring.toInteger() + integer = integer % (MAXVERSIONNUMBER + 1) + integers.add(integer) + } + def versionprefix = "" - if (splits.size() < 1) { + if (integers.size() < 1) { echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" versionprefix = defaultversionprefix } else { - def int splitnumbercount = 0 - for (split in splits) { - if (split.isInteger()) { - versionprefix += split + "." - splitnumbercount += 1 - } - else if (splitnumbercount > 2) { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { break } } - if (splitnumbercount == 0) { + if (integernumbercount == 0) { echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" versionprefix = defaultversionprefix } - else { - if (splitnumbercount == 1) { - versionprefix += "0.0." - } - else if (splitnumbercount == 2) { - versionprefix += "0." - } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." } } echo "Determined version prefix to be: ${versionprefix}" From 3bad7e2597a9242c529a66b3dbb2949de572ac79 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 21:36:36 +0000 Subject: [PATCH 142/195] Parameterise the release repo --- autobuilder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index bbe7bb9..732e268 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -40,7 +40,9 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - + + meta.releaseinfo.set(releaserepo) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 3eb169c9da50d78f63ac92c33a7a8fd585049ef0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:20:04 +0000 Subject: [PATCH 143/195] Check for existence of release repo by checking it out --- autobuilder.groovy | 3 ++- release/registerrepo.groovy | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 release/registerrepo.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 732e268..2a04925 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -41,7 +41,8 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - meta.releaseinfo.set(releaserepo) + def rr = load("${commonscripts()}/release/registerrepo.groovy") + rr.registerrepo(meta, releaserepo) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..1fc76ab --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo) { + if (releaserepo?.trim()) { + vc.gitCheckout(releaserepo, "origin/master", "meta/releaserepo") + } + meta.releaseinfo.set(releaserepo) +} + +return this; From d297db525283febc1e133fb294a0a66b94e1d372 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:32:50 +0000 Subject: [PATCH 144/195] Save the temp repo load in a tmp folder --- release/registerrepo.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index 1fc76ab..ff4d85d 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -4,7 +4,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data def registerrepo(meta, String releaserepo) { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "meta/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") } meta.releaseinfo.set(releaserepo) } From 0a7a6673bcb7f21a84fb37fb85f15aa846c2591e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:59:38 +0000 Subject: [PATCH 145/195] Make compile job use meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 28fe85f..0f9e0d5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -138,8 +138,8 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix private int revision - private String buildstamp - private String buildstampcommasep + String buildstamp + String buildstampcommasep BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { super(script) From c9a8032b36143cd19a38416561047a8948afbd6e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 11:24:37 +0000 Subject: [PATCH 146/195] Rename compile_init --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0f9e0d5..189014c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -162,7 +162,7 @@ class BuildStampInfo extends GISJsonSerialised { buildstamp = rawmeta['buildstamp'] buildstampcommasep = rawmeta['buildstampcommasep'] } - + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, From b5d80d67e8e3f561ca4e4c56db73cfc84fcce77d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 12:34:53 +0000 Subject: [PATCH 147/195] Correct checksumset deserialisation --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 189014c..e9491fe 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -253,7 +253,7 @@ class ChecksumPair extends GISJsonSerialised { private String filepath private String sum - ChecksumPair(script, String filepath, String sum) { + ChecksumPair(script, String filepath = "", String sum = "") { super(script) this.filepath = filepath this.sum = sum @@ -277,7 +277,7 @@ class ChecksumSet extends GISJsonSerialised { private String artifactpath private ArrayList checksumpairs - ChecksumSet(script, artifactpath) { + ChecksumSet(script, artifactpath = "") { super(script) this.artifactpath = artifactpath checksumpairs = new ArrayList() From 3752aa82ca371a5e4f278fb1821587265e580cf8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 13:18:35 +0000 Subject: [PATCH 148/195] Move image create job to use meta --- meta/folder.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index e9491fe..8f26265 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -507,11 +507,15 @@ class MetaInfo extends GISJsonSerialised { return path } + String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) def String adjustedpath = relativize(destpath) - def importedmetafile = adjustedpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) } From f50bbb36abbcc4e49af0c919afa51b706970e4dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 14:14:13 +0000 Subject: [PATCH 149/195] Update build job to meta --- artifacts/metaexport.groovy | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 66a8153..0cfc773 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,21 +1,19 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(subJobs) { - artifactFolders = "" - for (subJob in subJobs) { - subJobObject = subJob[0] - artifactFolder = subJob[1] - if (subJobObject != null) { - subJobName = subJobObject.getProjectName() - subJobNumber = subJobObject.getNumber() - destFolder = "${artifactFolder}" +def combineSubjobArtifacts(meta, subJobs) { + def im = load("${commonscripts()}/import/importer.groovy") - dir("${destFolder}") { + def artifactFolders = "" + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getProjectName() + def subJobNumber = subJobObject.getNumber() + destFolder = subJob[1] + dir(destFolder) { deleteDir() } - - echo "Copying artifacts from: Job: ${subJobName} Number: ${subJobNumber} to folder: ${destFolder}" - copyArtifacts fingerprintArtifacts: true, projectName: "${subJobName}", selector: specific("${subJobNumber}"), target: "${destFolder}" + im.artifactimport(meta, subJobName, specific(subJobNumber), destFolder) artifactFolders += "${destFolder}/**," } } From ee97d816c8cbb5de9f4224a34c48a525ebb048ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 25 Nov 2018 17:01:32 +0000 Subject: [PATCH 150/195] Update build job to meta --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0cfc773..016add9 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,7 +1,7 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, subJobs) { - def im = load("${commonscripts()}/import/importer.groovy") + def im = load("${autobuilder.commonscripts()}/import/importer.groovy") def artifactFolders = "" for (subJob in subJobs) { From 36b2d8bd90195a63d8864c1deba918e632fec38c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 25 Nov 2018 23:19:27 +0000 Subject: [PATCH 151/195] Make release job use meta --- artifacts/metaexport.groovy | 4 ++-- import/importer.groovy | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 016add9..62a6ba4 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,7 +1,7 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, subJobs) { - def im = load("${autobuilder.commonscripts()}/import/importer.groovy") + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def artifactFolders = "" for (subJob in subJobs) { @@ -13,7 +13,7 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - im.artifactimport(meta, subJobName, specific(subJobNumber), destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders += "${destFolder}/**," } } diff --git a/import/importer.groovy b/import/importer.groovy index d3c97e4..a9e28b2 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,15 @@ +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: specific(buildnumber.toString()), + target: destpath + ) + meta.importrelated(destpath) +} + def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -13,4 +24,4 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat meta.importrelated(destpath) } -return this; \ No newline at end of file +return this; From 9106a00c932747ee5fd0a6baf4cf95d811b60c71 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 08:29:56 +0000 Subject: [PATCH 152/195] Build up list of artifacts --- artifacts/metaexport.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 62a6ba4..cf5e9b3 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -3,7 +3,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def artifactFolders = "" + def artifactFolders = [] for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,7 +14,7 @@ def combineSubjobArtifacts(meta, subJobs) { deleteDir() } importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - artifactFolders += "${destFolder}/**," + artifactFolders.add("${destFolder}/**") } } return artifactFolders; From 0150bf139f8e29a8816c786a4ea01f8bce10ecf6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 11:16:53 +0000 Subject: [PATCH 153/195] Explicitly return artifact list as ArrayList --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index cf5e9b3..b6a832b 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -3,7 +3,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def artifactFolders = [] + def ArrayList artifactFolders = new ArrayList() for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { From 7e58b0b5a1babecf042aafc4e79651b8819658d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 18:17:55 +0000 Subject: [PATCH 154/195] Walk meta data for release info --- import/importer.groovy | 4 ++-- meta/folder.groovy | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index a9e28b2..4d24d30 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -7,7 +7,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +21,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f26265..c2c30a4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -175,8 +175,8 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private String releaserepoURL - private ArrayList releaseitems + String releaserepoURL + ArrayList releaseitems ReleaseInfo(script, releaserepoURL = "") { super(script) @@ -563,6 +563,26 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } + + // Recurse tree of meta looking for release info and accumulate path + private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { + if (releaseinfo.releaserepoURL) { + ArrayList rmi(script, pathsofar, this) + releasemetacollection.add(rmi) + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath + releasemetacollection = releasewalkerp(releasemetacollection, newpathsofar) + } + return releasemetacollection + } + + ArrayList releasewalker(String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(releasemetacollection, startpath) + return releasemetacollection + } } MetaInfo metareset(script) { From 1dffa1ef60eb6f9b4da17597dfee0d6992e4b612 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 18:37:54 +0000 Subject: [PATCH 155/195] Walk meta data for release info --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index c2c30a4..361c120 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -567,7 +567,7 @@ class MetaInfo extends GISJsonSerialised { // Recurse tree of meta looking for release info and accumulate path private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { if (releaseinfo.releaserepoURL) { - ArrayList rmi(script, pathsofar, this) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) releasemetacollection.add(rmi) } for (relatedmetainfo in associatedmeta) { From 68f24e6a30a3f8330a6e5914bb99bd69fc153fbf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 21:18:12 +0000 Subject: [PATCH 156/195] Walk meta data for release info --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 361c120..b204462 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -573,7 +573,7 @@ class MetaInfo extends GISJsonSerialised { for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath - releasemetacollection = releasewalkerp(releasemetacollection, newpathsofar) + releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } return releasemetacollection } From 84589b75474c3189c35c18320fc13da0aff4649f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 08:34:19 +0000 Subject: [PATCH 157/195] Correct release path --- meta/folder.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index b204462..88c5e17 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -572,7 +572,10 @@ class MetaInfo extends GISJsonSerialised { } for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath + if (pathsofar) { + pathsofar += "/" + } + String newpathsofar = pathsofar + relatedmetainfo.offsetpath releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } return releasemetacollection From 6211375100f8e0f029d653d1e7d3851bbe19261f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 08:51:27 +0000 Subject: [PATCH 158/195] Correct release path --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 88c5e17..802443d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -570,11 +570,11 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) releasemetacollection.add(rmi) } + if (pathsofar) { + pathsofar += "/" + } for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta - if (pathsofar) { - pathsofar += "/" - } String newpathsofar = pathsofar + relatedmetainfo.offsetpath releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } From da2e95a8705d2ead0a4e2c9b5f05dd592345226c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 16:52:38 +0000 Subject: [PATCH 159/195] Check out release repos --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 802443d..f5e7c93 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,7 +564,7 @@ class MetaInfo extends GISJsonSerialised { return artifactlist } - // Recurse tree of meta looking for release info and accumulate path + // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { if (releaseinfo.releaserepoURL) { RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) From 976994accc03b3374a4de0efa06fa0b438ea5b3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 18:09:27 +0000 Subject: [PATCH 160/195] Check out release repos --- meta/folder.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f5e7c93..1e45319 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -507,7 +507,7 @@ class MetaInfo extends GISJsonSerialised { return path } - String metafilepath() { + static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } @@ -598,4 +598,9 @@ MetaInfo metareset(script) { return mi } +MetaInfo metafromfolder(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") +} + return this; From 03921f73d4226056021efb821daaf18eec6445dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 18:47:42 +0000 Subject: [PATCH 161/195] Check out release repos --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1e45319..de445f1 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -376,10 +376,10 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private String FORMAT = "0.1" - private String METAFOLDERNAME = "meta" - private String METAFILENAME = "info.json" - private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String FORMAT = "0.1" + private static String METAFOLDERNAME = "meta" + private static String METAFILENAME = "info.json" + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath From 0ee39e5c95e1b9b2d1beba62b40548bb4258ea7e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 19:10:54 +0000 Subject: [PATCH 162/195] Check out release repos --- autobuilder.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 2a04925..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -25,17 +27,21 @@ def getautobuildpath() { return autobuildroot } +def getmetafolderscript() { + return metafolderscript +} + // Start up boilderplate def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - def meta = metafolder.metareset(script) + def meta = getmetafolderscript().metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) From 6a08d9ef26061b9c6d97afa2a3597aa3acd69c89 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:19:18 +0000 Subject: [PATCH 163/195] Check out release repos --- autobuilder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..885d243 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,6 +28,7 @@ def getautobuildpath() { } def getmetafolderscript() { + echo "metafolderscript is type: ${metafolderscript.toClass()}" return metafolderscript } From 3c8510bb275a21aa1a883eca2dcc99c9c0b49f7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:23:17 +0000 Subject: [PATCH 164/195] Check out release repos --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 885d243..e40a67e 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,7 +28,7 @@ def getautobuildpath() { } def getmetafolderscript() { - echo "metafolderscript is type: ${metafolderscript.toClass()}" + echo "metafolderscript is type: ${metafolderscript.getClass()}" return metafolderscript } From e7085c9e219608078132ee5b65423a2c0f0f401c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:41:01 +0000 Subject: [PATCH 165/195] Check out release repos --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index de445f1..458156d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -601,6 +601,7 @@ MetaInfo metareset(script) { MetaInfo metafromfolder(String folder = "") { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta } return this; From fca9f9f00d48ff56eb76fcbeeb27914af5fe1246 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 22:35:37 +0000 Subject: [PATCH 166/195] Remove debug --- autobuilder.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index e40a67e..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,7 +28,6 @@ def getautobuildpath() { } def getmetafolderscript() { - echo "metafolderscript is type: ${metafolderscript.getClass()}" return metafolderscript } From 3f8bb3f6f7c7de23558eaa4111dc8de2c1a33fcc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 09:07:06 +0000 Subject: [PATCH 167/195] Create a version number for the all build --- artifacts/metaexport.groovy | 8 ++++++-- meta/folder.groovy | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b6a832b..e87e3d9 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,6 +4,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() + int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -13,11 +14,14 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + revisiontotal += importedmeta.buildstampinfo.revision artifactFolders.add("${destFolder}/**") } } - return artifactFolders; + def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + revisiontotal = revisiontotal % (versiongenerator.MAXVERSIONNUMBER + 1) + return [artifactFolders, revisiontotal] } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 458156d..74ac7c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -137,7 +137,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix - private int revision + int revision String buildstamp String buildstampcommasep @@ -156,6 +156,14 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + void changerevision(int newrevision) { + revision = newrevision + buildstampsplit = buildstamp.split('.') + buildstampsplit[3] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + buildstampcommasep = buildstampsplit.join(',') + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From 1a4666fdcce33377839c1992fb2f6410a2cd877a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 11:37:10 +0000 Subject: [PATCH 168/195] Correct scope of MAXVERSIONNUMBER --- versionnumbering/generate.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 59fe2a3..cf21fd0 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,3 +1,4 @@ +int MAXVERSIONNUMBER = 65535 def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount @@ -5,8 +6,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) @@ -28,8 +27,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { - def int MAXVERSIONNUMBER = 65535 - echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() From 46ddde1b777ab67847f7ad5de4cab37ca6488143 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 13:10:26 +0000 Subject: [PATCH 169/195] Correct scope of MAXVERSIONNUMBER --- artifacts/metaexport.groovy | 2 +- versionnumbering/generate.groovy | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index e87e3d9..b4d4a6d 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -20,7 +20,7 @@ def combineSubjobArtifacts(meta, subJobs) { } } def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.MAXVERSIONNUMBER + 1) + revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) return [artifactFolders, revisiontotal] } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cf21fd0..9df06d8 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,4 +1,8 @@ -int MAXVERSIONNUMBER = 65535 + +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount @@ -8,7 +12,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -62,7 +66,7 @@ def getVersionPrefix(String sourcebranch) { def integers = [] for (integerstring in integerstrings) { int integer = integerstring.toInteger() - integer = integer % (MAXVERSIONNUMBER + 1) + integer = integer % (maxversionnumber() + 1) integers.add(integer) } From 58b6100215be6ed6ddbe3eb7abacaacac88fff50 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:11:08 +0000 Subject: [PATCH 170/195] Load in artifacts for release --- meta/folder.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 74ac7c4..822472d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -594,6 +594,17 @@ class MetaInfo extends GISJsonSerialised { releasemetacollection = releasewalkerp(releasemetacollection, startpath) return releasemetacollection } + + Map releasewalkerunique(String startpath = "") { + Map releasemetauniquecollection = new Map() + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(releasemetacollection, startpath) + for (relatedmetainfo in releasemetacollection) { + String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetauniquecollection + } } MetaInfo metareset(script) { From c095a2c19ef6afcb69febebfb1a5b5e20392a9d3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:17:09 +0000 Subject: [PATCH 171/195] Load in artifacts for release --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 822472d..8f8ea0a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -596,7 +596,7 @@ class MetaInfo extends GISJsonSerialised { } Map releasewalkerunique(String startpath = "") { - Map releasemetauniquecollection = new Map() + Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { From 911d147f81b3c3a3712490e8aca945e96b6c2118 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:40:47 +0000 Subject: [PATCH 172/195] Load in artifacts for release --- meta/folder.groovy | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f8ea0a..24bcbf5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -573,32 +573,34 @@ class MetaInfo extends GISJsonSerialised { } // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java - private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { - if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) - releasemetacollection.add(rmi) - } - if (pathsofar) { - pathsofar += "/" - } - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + relatedmetainfo.offsetpath - releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) + private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { + if (maxdepth > 0) { + if (releaseinfo.releaserepoURL) { + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + releasemetacollection.add(rmi) + } + if (pathsofar) { + pathsofar += "/" + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + relatedmetainfo.offsetpath + releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) + } } return releasemetacollection } - ArrayList releasewalker(String startpath = "") { + ArrayList releasewalker(int maxdepth, String startpath = "") { ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(releasemetacollection, startpath) + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) return releasemetacollection } - Map releasewalkerunique(String startpath = "") { + Map releasewalkerunique(int maxdepth, String startpath = "") { Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(releasemetacollection, startpath) + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname releasemetauniquecollection.put(uniquestring, relatedmetainfo) From 1acfec14d58fccd275ea435ef4bae6423aa5dde1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 19:24:10 +0000 Subject: [PATCH 173/195] Perform git push operations --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 24bcbf5..b8120a6 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -158,7 +158,7 @@ class BuildStampInfo extends GISJsonSerialised { void changerevision(int newrevision) { revision = newrevision - buildstampsplit = buildstamp.split('.') + def List buildstampsplit = buildstamp.split('.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') From a43f93cc15a854ff0425b36a355d6acaa5b614e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 19:44:11 +0000 Subject: [PATCH 174/195] Perform git push operations --- meta/folder.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index b8120a6..cfa0a6e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,11 +157,13 @@ class BuildStampInfo extends GISJsonSerialised { } void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") revision = newrevision def List buildstampsplit = buildstamp.split('.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') + script.echo("Build stamp now ${buildstamp}") } void deserialiseJson(rawmeta) { From 95b1a75e0c8c32161cac0ff241e32f0d9aedf5c7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 00:33:39 +0000 Subject: [PATCH 175/195] Calculate version number earlier --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cfa0a6e..ed17a91 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -159,7 +159,7 @@ class BuildStampInfo extends GISJsonSerialised { void changerevision(int newrevision) { script.echo("Build stamp currently ${buildstamp}") revision = newrevision - def List buildstampsplit = buildstamp.split('.') + def List buildstampsplit = buildstamp.split('\\.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') From 24b138ac8ffa91bc3bfee0403212417f8cb0b1de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:06:46 +0000 Subject: [PATCH 176/195] Remote URL fetch was not working --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ed17a91..2640567 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,4 +1,4 @@ -import java.nio.file.Paths +urimport java.nio.file.Paths import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 9df06d8..f185c40 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -21,7 +21,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 3fceb3282905b6cef5137eacf9cb0a17774e1d47 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:10:04 +0000 Subject: [PATCH 177/195] Remote URL fetch was not working --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2640567..ed17a91 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,4 +1,4 @@ -urimport java.nio.file.Paths +import java.nio.file.Paths import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { From 347e544b459cb5206c0145bc3007288412a062f3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:36:20 +0000 Subject: [PATCH 178/195] Remove comma separated buildstamp store --- meta/folder.groovy | 17 ++++++++--------- versionnumbering/generate.groovy | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ed17a91..76fbe43 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -139,30 +139,31 @@ class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix int revision String buildstamp - String buildstampcommasep - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } - void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } + String buildstampcommasep() { + buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + void changerevision(int newrevision) { script.echo("Build stamp currently ${buildstamp}") revision = newrevision def List buildstampsplit = buildstamp.split('\\.') - buildstampsplit[3] = newrevision.toString() + buildstampsplit[-1] = newrevision.toString() buildstamp = buildstampsplit.join('.') - buildstampcommasep = buildstampsplit.join(',') script.echo("Build stamp now ${buildstamp}") } @@ -170,7 +171,6 @@ class BuildStampInfo extends GISJsonSerialised { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] - buildstampcommasep = rawmeta['buildstampcommasep'] } LinkedHashMap serialisationdata() { @@ -178,7 +178,6 @@ class BuildStampInfo extends GISJsonSerialised { 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, - 'buildstampcommasep' : buildstampcommasep, ] return data } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f185c40..a7b583a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -113,7 +113,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum echo "Build stamp is: ${buildstamp}" meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) return buildstamp } From 574c2155a776b67f164ab426da7fcbe606a333f8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 12:48:41 +0000 Subject: [PATCH 179/195] Correct buildstamp comma sep generation --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 76fbe43..570a3ea 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -154,7 +154,7 @@ class BuildStampInfo extends GISJsonSerialised { } String buildstampcommasep() { - buildstampcommasep = buildstamp.replace(".", ",") + def String buildstampcommasep = buildstamp.replace(".", ",") return buildstampcommasep } From d33e50f3cc7779f41f6703b52ef97c8e73e62b78 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 13:23:40 +0000 Subject: [PATCH 180/195] Remove redundant script; Correct artifact export of release job --- versionnumbering/metaversion.groovy | 100 ---------------------------- 1 file changed, 100 deletions(-) delete mode 100644 versionnumbering/metaversion.groovy diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy deleted file mode 100644 index 44bfbde..0000000 --- a/versionnumbering/metaversion.groovy +++ /dev/null @@ -1,100 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - -def prefixline(rootpath, filepath) { - def fileContents = readFile("${filepath}").trim() - def fileContentsNormalized = fileContents.normalize() - - def newFileContents = "" - def lines = fileContentsNormalized.readLines() - for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + "${nl}" - } - return newFileContents -} - -def mergemeta(rootpath, versionFolder, outputFolder) { - echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents - - def intrevision = 0 - if (fileExists("${versionFolder}/revision.txt")) { - def revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - } - else { - intrevision = 1 - } - - return intrevision -} - -def appendcomponentinfo(rootpath, versionFolder, outputFolder) { - echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents - } - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = readFile("${buildStampFile}").trim() - - def remoteurlFile = "${versionFolder}/remoteurl.txt" - def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - - def gitRevisionFile = "${versionFolder}/gitrevision.txt" - def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - def releaseJobNameFile = "${versionFolder}/releasejobname.txt" - def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - def revision = readFile("${versionFolder}/revision.txt").trim() - def intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - - return intrevision -} - -// Each version folder should have been created using version creation scripts in this Common_scripts folder -// as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - def totalrevision = 0 - dir("${outputFolder}") { - deleteDir() - } - for (artifactItem in artifactList) { - (rootpath, versionFolder) = artifactItem - echo "Inspecting folder: ${versionFolder}" - def metaversionsignaturefile = "${versionFolder}/metaversion.txt" - // If the sub project is a meta job - def intrevision = 0 - if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta(rootpath, versionFolder, outputFolder) - } - else { - intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) - } - totalrevision += intrevision - } - echo "Accumulated revision is: ${totalrevision}" - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) -} - -return this; \ No newline at end of file From cc99fcf2a17aff04d651b5a24338cb8240476c9c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:03:12 +0000 Subject: [PATCH 181/195] Alter differences file generation; Merge differences when importing --- differences/do.groovy | 6 +++--- differences/show_release_differences.py | 11 +++++------ meta/folder.groovy | 22 +++++++++++++++++++--- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 35e0980..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,16 +1,16 @@ def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { def String cs = autobuilder.commonscripts() - def String difffolder = meta.folder() + def String difffile = meta.difffilepath() def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0813356..fee6f7b 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -172,7 +172,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") + parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +183,13 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - difffolder = args.difffolder + difffile = args.difffile print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - if difffolder: - makeFolder(difffolder) - outfh = open(os.path.join(difffolder, "differences.txt"), 'w') + if difffile: + outfh = open(difffile), 'w') else: outfh = sys.stdout diff --git a/meta/folder.groovy b/meta/folder.groovy index 570a3ea..8bd80d3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,8 +19,8 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - private String productname - private String subcomponentname + String productname + String subcomponentname String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { @@ -388,6 +388,7 @@ class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" private static String METAFILENAME = "info.json" + private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { @@ -519,7 +520,11 @@ class MetaInfo extends GISJsonSerialised { static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } - + + static String difffilepath() { + return METAFOLDERNAME + "/" + METADIFFFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) @@ -532,6 +537,17 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) + def importedmetadifffile = adjustedpath + "/" + difffilepath() + if (script.fileExists(importedmetadifffile)) { + def String newcontents = script.readFile(file: importedmetadifffile) + def String currentcontents = "" + if (script.fileExists(difffilepath())) { + currentcontents = script.readFile(file: difffilepath()) + } + + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " mi.productinfo.componentname + ":\n" + newcontents) + } + return mi } From e452d7823e3bb03a2c30157175514cdfb5c9a389 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:05:23 +0000 Subject: [PATCH 182/195] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8bd80d3..2269a77 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " mi.productinfo.componentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.componentname + ":\n" + newcontents) } return mi From c98a09f5250d499c22f3c0b426213151cee2ce44 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:31:21 +0000 Subject: [PATCH 183/195] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2269a77..ac2ab9a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.componentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) } return mi From 0f2ae8596496efab7e0bf0448bac9ec93e494346 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:36:42 +0000 Subject: [PATCH 184/195] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ac2ab9a..9c5367e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) } return mi From 94ef8956ba6bdd56bd225850ae290e2c0771dfde Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:41:49 +0000 Subject: [PATCH 185/195] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index fee6f7b..0600753 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -189,7 +189,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: - outfh = open(difffile), 'w') + outfh = open(difffile, 'w') else: outfh = sys.stdout From 5491405110db56d84eedeb45368523c1d2793721 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:51:43 +0000 Subject: [PATCH 186/195] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0600753..ddf8c0d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,10 +29,6 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(difffolder): - print("Creating folder (if not already there): {0}".format(difffolder)) - os.makedirs(difffolder, exist_ok = True) - def printDivider(outfh, times = 1, length = 100): for _ in range(times): outfh.write("{0}\n".format("*" * length)) @@ -208,7 +204,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if difffolder: + if difffile: outfh.close() return 0 except Exception as e: From 5bb08476af818e51c87b40798049bf249bc2b6a4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 15:05:23 +0000 Subject: [PATCH 187/195] Add HMBFDGMA jira comment regular expression --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index ddf8c0d..80b636e 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -80,6 +80,9 @@ jiraprojectpattern = { 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), + 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From aa497116fd9078857227afef34bf88c184030998 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 15:19:55 +0000 Subject: [PATCH 188/195] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 80b636e..5c0f15d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -188,6 +188,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: + os.makedirs(os.path.dirname(difffile), exist_ok=True) outfh = open(difffile, 'w') else: outfh = sys.stdout From db54da7b0577f004033c5104c5f22fc10f090fdc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 17:48:05 +0000 Subject: [PATCH 189/195] Move release functionality into Common_scripts --- release/do.groovy | 131 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 release/do.groovy diff --git a/release/do.groovy b/release/do.groovy new file mode 100644 index 0000000..72c2f9f --- /dev/null +++ b/release/do.groovy @@ -0,0 +1,131 @@ +boolean markunstable = false +String unstablereason = "" + +String summaryfolder = "${WORKSPACE}/summary" +String summaryfile = "${summaryfolder}/summary.txt" +String releaserepofolder = "${WORKSPACE}/releaserepos" +String tempfolder = "${WORKSPACE}/temp" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def start() { + markunstable = false + unstablereason = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary + "\n") +} + +def checkoutreleaserepo(info, String checkoutfolder) { + def String releaserepo = info.meta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component + dir(releaserepofolder) { + sh("git clone -n --depth 1 ${releaserepo} ${checkoutfolder}") + } +} + +def getcomponentartifacts(info, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + for (artifactfilter in info.meta.releaseinfo.releaseitems) { + def filterpath = artifactfilter + if (info.offsetpath) { + filterpath = info.offsetpath + '/' + artifactfilter + } + // Copy relevant artifacts to temp folder + echo "Copying artifacts: ${filterpath} to ${tempfolder}" + copyArtifacts( + filter: filterpath, + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: tempfolder + ) + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + info.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } + summaryappend("${artifactfilter},") + } +} + +def gitrelease(String releaserepo, String buildstamp) { + summaryappend(" Git operations: ") + dir(releaserepo) { + // Add anything new + sh("git add .") + summaryappend("Add,") + // Perform git commit (if necessary) + def changesoccurred = sh(script: "git status --porcelain", returnStdout: true) + if (changesoccurred) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") + } + else { + summaryappend("No-commit,") + } + // Tag with buildstamp + sh("git tag ${buildstamp}") + summaryappend("Tag (${buildstamp}),") + if (DRY_RUN_PUSH == "true") { + sh("git push -n --tags origin master") + summaryappend("Dry-Run-Push,") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend("Push") + } + } +} + +def releasecomponent(info) { + def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${info.meta.productinfo.productname} ${info.meta.productinfo.subcomponentname}: ") + checkoutreleaserepo(info, uniquefolderforrepo) + + getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) + + def String buildstamp = info.meta.buildstampinfo.buildstamp + gitrelease(uniquefolderforrepo, buildstamp) +} + +def releasehierarchy(buildmeta, int maxdepth = MAX_META_DEPTH) { + start() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = buildmeta.releasewalkerunique(maxdepth.toInteger()) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def info = componentReleaseInfo[key] + releasecomponent(info) + } + return [markunstable, unstablereason] +} + +return this; \ No newline at end of file From 66973f0034206d128ecdbd661410425d0bd4c999 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 17:56:08 +0000 Subject: [PATCH 190/195] Move release functionality into Common_scripts --- release/do.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 72c2f9f..239cc17 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -114,11 +114,12 @@ def releasecomponent(info) { gitrelease(uniquefolderforrepo, buildstamp) } -def releasehierarchy(buildmeta, int maxdepth = MAX_META_DEPTH) { +def releasehierarchy(buildmeta, String maxdepth) { + int md = maxdepth.toInteger() start() // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = buildmeta.releasewalkerunique(maxdepth.toInteger()) + componentReleaseInfo = buildmeta.releasewalkerunique(md) // For each component for (key in componentReleaseInfo.keySet()) { From f724c2115209e3d049b3e7bc381cbf5ba5609e34 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:09:13 +0000 Subject: [PATCH 191/195] Move release functionality into Common_scripts --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 239cc17..17090fe 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -1,10 +1,10 @@ -boolean markunstable = false -String unstablereason = "" +markunstable = false +unstablereason = "" -String summaryfolder = "${WORKSPACE}/summary" -String summaryfile = "${summaryfolder}/summary.txt" -String releaserepofolder = "${WORKSPACE}/releaserepos" -String tempfolder = "${WORKSPACE}/temp" +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +releaserepofolder = "${WORKSPACE}/releaserepos" +tempfolder = "${WORKSPACE}/temp" def resetfolders() { dir(summaryfolder) { From e17df02c5cb1017411d435c7b061365ea7fd287f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:11:37 +0000 Subject: [PATCH 192/195] Move release functionality into Common_scripts --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index 17090fe..177593d 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -118,6 +118,7 @@ def releasehierarchy(buildmeta, String maxdepth) { int md = maxdepth.toInteger() start() + resetfolders() // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) From 5d53126d199b99a070b4ebbf7c58b7623ea5535a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:15:32 +0000 Subject: [PATCH 193/195] Move release functionality into Common_scripts --- release/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 177593d..1eaf2b8 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -18,7 +18,7 @@ def resetfolders() { } } -def start() { +def preconditions() { markunstable = false unstablereason = "" writeFile(file: summaryfile, text: "Release summary: \n") @@ -116,9 +116,9 @@ def releasecomponent(info) { def releasehierarchy(buildmeta, String maxdepth) { int md = maxdepth.toInteger() - start() - resetfolders() + preconditions() + // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) From 8316cf4cf71f0dba161436441509e0cd168235c3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:19:59 +0000 Subject: [PATCH 194/195] Move release functionality into Common_scripts --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 1eaf2b8..264bd53 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -70,7 +70,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { } } -def gitrelease(String releaserepo, String buildstamp) { +def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { // Add anything new @@ -88,7 +88,7 @@ def gitrelease(String releaserepo, String buildstamp) { // Tag with buildstamp sh("git tag ${buildstamp}") summaryappend("Tag (${buildstamp}),") - if (DRY_RUN_PUSH == "true") { + if (dryrunpush == "true") { sh("git push -n --tags origin master") summaryappend("Dry-Run-Push,") setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") @@ -100,7 +100,7 @@ def gitrelease(String releaserepo, String buildstamp) { } } -def releasecomponent(info) { +def releasecomponent(info, String dryrunpush) { def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -111,10 +111,10 @@ def releasecomponent(info) { getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) def String buildstamp = info.meta.buildstampinfo.buildstamp - gitrelease(uniquefolderforrepo, buildstamp) + gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) } -def releasehierarchy(buildmeta, String maxdepth) { +def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { int md = maxdepth.toInteger() resetfolders() preconditions() @@ -125,7 +125,7 @@ def releasehierarchy(buildmeta, String maxdepth) { // For each component for (key in componentReleaseInfo.keySet()) { def info = componentReleaseInfo[key] - releasecomponent(info) + releasecomponent(info, dryrunpush) } return [markunstable, unstablereason] } From e4f8dab2bbee21d2504f8466f2f60cd9eb00717b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:39:53 +0000 Subject: [PATCH 195/195] Move release functionality into Common_scripts --- release/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 264bd53..6db301c 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -31,7 +31,7 @@ def setunstable(String reason) { def summaryappend(String summary) { def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary + "\n") + writeFile(file: summaryfile, text: currentsummarycontents + summary) } def checkoutreleaserepo(info, String checkoutfolder) { @@ -112,6 +112,8 @@ def releasecomponent(info, String dryrunpush) { def String buildstamp = info.meta.buildstampinfo.buildstamp gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) + + summaryappend("\n") } def releasehierarchy(buildmeta, String maxdepth, dryrunpush) {