From 8d73541b51b48be36b26edc337bbf7b52538d76f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Oct 2018 14:53:02 +0100 Subject: [PATCH 001/589] Using SM200 master (714fd686bf5ea07f83346342362eca45c6ffc9df) as a starting point for HMB-FD-HV autobuild scripts --- .project | 17 ++ .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + commandline/commandline.gsh | 13 ++ .../eclipsebuild/disableparallelbuilds.gsh | 12 + compile/eclipsebuild/eclipseds5build.gsh | 180 ++++++++++++++ configuration/configuration.gsh | 37 +++ differences/.project | 17 ++ differences/.pydevproject | 8 + differences/show_release_differences.py | 220 ++++++++++++++++++ test.gsh | 7 + versioncontrol/git.gsh | 63 +++++ versionnumbering/versioninfo.py | 219 +++++++++++++++++ versionstamping/.project | 17 ++ versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 41 ++++ 16 files changed, 869 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 commandline/commandline.gsh create mode 100644 compile/eclipsebuild/disableparallelbuilds.gsh create mode 100644 compile/eclipsebuild/eclipseds5build.gsh create mode 100644 configuration/configuration.gsh create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/show_release_differences.py create mode 100644 test.gsh create mode 100644 versioncontrol/git.gsh create mode 100644 versionnumbering/versioninfo.py create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/commandline/commandline.gsh b/commandline/commandline.gsh new file mode 100644 index 0000000..df99a75 --- /dev/null +++ b/commandline/commandline.gsh @@ -0,0 +1,13 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return bat(returnStdout: true, script: """ + @echo off + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py new file mode 100644 index 0000000..93d540b --- /dev/null +++ b/differences/show_release_differences.py @@ -0,0 +1,220 @@ +''' +showreleasedifferences -- shortdesc + +showreleasedifferences generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import subprocess +import os +import sys +import re + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") + +class GitException(Exception): + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + +def makeFolder(versionfolder): + print("Creating folder (if not already there): {0}".format(versionfolder)) + os.makedirs(versionfolder, exist_ok = True) + +def printDivider(outfh, times = 1, length = 100): + for _ in range(times): + outfh.write("{0}\n".format("*" * length)) + +TABSIZE = 4 +def printTabbed(outfh, string, tabs = 0): + outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) + +def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): + if tripledot: + releaseExpression = previousRelease + "..." + currentRelease + else: + releaseExpression = previousRelease + ".." + currentRelease + currentdir = os.getcwd() + os.chdir(gitrootfolder) + gitlog = [] + printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if line: + linestring = line.decode('utf-8').rstrip() + gitlog.append(linestring) + #printTabbed(outfh, linestring, 1) + else: + break + proc.wait() + if proc.returncode > 0: + raise GitException("Git executable returned an error") + os.chdir(currentdir) + return gitlog + +jiraprojectpattern = { + 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), + ), + 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), + ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), + } + +mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") + +def processLog(gitlog, jirakeyprefix): + reverts = [] + merges = {} + other = [] + numbered = {} + (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] + for line in gitlog: + expectedmatch = expectedpattern.match(line) + revertmatch = revertpattern.match(line) + mergematch = mergepattern.match(line) + if expectedmatch == None and revertmatch == None and mergematch == None: + other.append(line) + if revertmatch: + gisnumber = jirakeyprefix + "-" + revertmatch.group(2) + description = revertmatch.group(3) + reverts.append("Revert: {0} {1}".format(gisnumber, description)) + if mergematch: + merge = mergematch.group(2) + if not merge in merges: + merges[merge] = 1 + else: + merges[merge] += 1 + if expectedmatch: + gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) + description = expectedmatch.group(2) + if not gisnumber in numbered: + numbered[gisnumber] = [] + numbered[gisnumber].append(description) + return (reverts, merges, other, numbered) + +def showDifferences(differences, jirakeyprefix, outfh): + (revertslog, mergeslog, otherlog, numberedlog) = differences + printDivider(outfh) + printTabbed(outfh, jirakeyprefix + "-Numbers Log:") + for (gisnumber, descriptions) in sorted(numberedlog.items()): + printTabbed(outfh, gisnumber, 1) + descriptions = numberedlog[gisnumber]; + for description in descriptions: + printTabbed(outfh, description, 2) + printDivider(outfh) + printTabbed(outfh, "Revert log:") + for reversion in sorted(revertslog): + printTabbed(outfh, reversion, 1) + printDivider(outfh) + printTabbed(outfh, "Merges log:") + for merge in sorted(mergeslog): + printTabbed(outfh, merge, 1) + printDivider(outfh) + printTabbed(outfh, "Other log:") + for other in sorted(otherlog): + printTabbed(outfh, other, 1) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") + parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") + parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") + parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + gitrootfolder = args.gitrootfolder + sourceversion = args.sourceversion + previousversion = args.previousversion + jirakeyprefix = args.jirakeyprefix + outputoptions = args.outputoptions + versionfolder = args.versionfolder + + print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + + if versionfolder: + makeFolder(versionfolder) + outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + else: + outfh = sys.stdout + + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'opposite' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'notcommon' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + + if versionfolder: + outfh.close() + return 0 + except Exception as e: + print("Error while determining differences:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..23fcfc0 --- /dev/null +++ b/test.gsh @@ -0,0 +1,7 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; + diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh new file mode 100644 index 0000000..dbc09ff --- /dev/null +++ b/versioncontrol/git.gsh @@ -0,0 +1,63 @@ +clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git log -1 --format=oneline') + } +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + ], + submoduleCfg: [], + ]) +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() <= 1) { + return false + } else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +return this diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..a9d8738 --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,219 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision, remoteinfo) + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return mastertypeversionprefix + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..3e59c6f --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,41 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + filecopy = filetoconvert + ".orig" + try: + shutil.move(filetoconvert, filecopy) + except FileNotFoundError: + print("Could not find file: {0}\n".format(filetoconvert)) + return 1 + + with open(filetoconvert, 'x') as f: + with open(filecopy,'r') as c: + fileread = c.read() + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + #print ("DEBUG: %s "%converted) + print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + f.write(converted) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 0512a83425de4e8a056b14fb8a5b2dc610201328 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 11 Oct 2018 20:11:30 +0100 Subject: [PATCH 002/589] HMBFDHV-12 - Add HMBFDHV differences search --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 93d540b..4937134 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -78,6 +78,9 @@ jiraprojectpattern = { 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), ), + 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From 35cd5c63c6a3dc3074552324c4c387a36d5534f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 15 Oct 2018 15:54:16 +0100 Subject: [PATCH 003/589] Merge commit 'fadf5e954325f20d50142038be5aa428ad5d31af' into production --- .project | 17 ++ .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + commandline/commandline.gsh | 13 ++ .../eclipsebuild/disableparallelbuilds.gsh | 12 + compile/eclipsebuild/eclipseds5build.gsh | 186 +++++++++++++++ configuration/configuration.gsh | 37 +++ differences/.project | 17 ++ differences/.pydevproject | 8 + differences/show_release_differences.py | 220 ++++++++++++++++++ test.gsh | 7 + versioncontrol/git.gsh | 63 +++++ versionnumbering/versioninfo.py | 219 +++++++++++++++++ versionstamping/.project | 17 ++ versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 41 ++++ 16 files changed, 875 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 commandline/commandline.gsh create mode 100644 compile/eclipsebuild/disableparallelbuilds.gsh create mode 100644 compile/eclipsebuild/eclipseds5build.gsh create mode 100644 configuration/configuration.gsh create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/show_release_differences.py create mode 100644 test.gsh create mode 100644 versioncontrol/git.gsh create mode 100644 versionnumbering/versioninfo.py create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/commandline/commandline.gsh b/commandline/commandline.gsh new file mode 100644 index 0000000..df99a75 --- /dev/null +++ b/commandline/commandline.gsh @@ -0,0 +1,13 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return bat(returnStdout: true, script: """ + @echo off + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py new file mode 100644 index 0000000..93d540b --- /dev/null +++ b/differences/show_release_differences.py @@ -0,0 +1,220 @@ +''' +showreleasedifferences -- shortdesc + +showreleasedifferences generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import subprocess +import os +import sys +import re + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") + +class GitException(Exception): + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + +def makeFolder(versionfolder): + print("Creating folder (if not already there): {0}".format(versionfolder)) + os.makedirs(versionfolder, exist_ok = True) + +def printDivider(outfh, times = 1, length = 100): + for _ in range(times): + outfh.write("{0}\n".format("*" * length)) + +TABSIZE = 4 +def printTabbed(outfh, string, tabs = 0): + outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) + +def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): + if tripledot: + releaseExpression = previousRelease + "..." + currentRelease + else: + releaseExpression = previousRelease + ".." + currentRelease + currentdir = os.getcwd() + os.chdir(gitrootfolder) + gitlog = [] + printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if line: + linestring = line.decode('utf-8').rstrip() + gitlog.append(linestring) + #printTabbed(outfh, linestring, 1) + else: + break + proc.wait() + if proc.returncode > 0: + raise GitException("Git executable returned an error") + os.chdir(currentdir) + return gitlog + +jiraprojectpattern = { + 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), + ), + 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), + ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), + } + +mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") + +def processLog(gitlog, jirakeyprefix): + reverts = [] + merges = {} + other = [] + numbered = {} + (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] + for line in gitlog: + expectedmatch = expectedpattern.match(line) + revertmatch = revertpattern.match(line) + mergematch = mergepattern.match(line) + if expectedmatch == None and revertmatch == None and mergematch == None: + other.append(line) + if revertmatch: + gisnumber = jirakeyprefix + "-" + revertmatch.group(2) + description = revertmatch.group(3) + reverts.append("Revert: {0} {1}".format(gisnumber, description)) + if mergematch: + merge = mergematch.group(2) + if not merge in merges: + merges[merge] = 1 + else: + merges[merge] += 1 + if expectedmatch: + gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) + description = expectedmatch.group(2) + if not gisnumber in numbered: + numbered[gisnumber] = [] + numbered[gisnumber].append(description) + return (reverts, merges, other, numbered) + +def showDifferences(differences, jirakeyprefix, outfh): + (revertslog, mergeslog, otherlog, numberedlog) = differences + printDivider(outfh) + printTabbed(outfh, jirakeyprefix + "-Numbers Log:") + for (gisnumber, descriptions) in sorted(numberedlog.items()): + printTabbed(outfh, gisnumber, 1) + descriptions = numberedlog[gisnumber]; + for description in descriptions: + printTabbed(outfh, description, 2) + printDivider(outfh) + printTabbed(outfh, "Revert log:") + for reversion in sorted(revertslog): + printTabbed(outfh, reversion, 1) + printDivider(outfh) + printTabbed(outfh, "Merges log:") + for merge in sorted(mergeslog): + printTabbed(outfh, merge, 1) + printDivider(outfh) + printTabbed(outfh, "Other log:") + for other in sorted(otherlog): + printTabbed(outfh, other, 1) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") + parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") + parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") + parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + gitrootfolder = args.gitrootfolder + sourceversion = args.sourceversion + previousversion = args.previousversion + jirakeyprefix = args.jirakeyprefix + outputoptions = args.outputoptions + versionfolder = args.versionfolder + + print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + + if versionfolder: + makeFolder(versionfolder) + outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + else: + outfh = sys.stdout + + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'opposite' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'notcommon' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + + if versionfolder: + outfh.close() + return 0 + except Exception as e: + print("Error while determining differences:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..23fcfc0 --- /dev/null +++ b/test.gsh @@ -0,0 +1,7 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; + diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh new file mode 100644 index 0000000..dbc09ff --- /dev/null +++ b/versioncontrol/git.gsh @@ -0,0 +1,63 @@ +clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git log -1 --format=oneline') + } +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + ], + submoduleCfg: [], + ]) +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() <= 1) { + return false + } else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +return this diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..a9d8738 --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,219 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision, remoteinfo) + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return mastertypeversionprefix + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..3e59c6f --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,41 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + filecopy = filetoconvert + ".orig" + try: + shutil.move(filetoconvert, filecopy) + except FileNotFoundError: + print("Could not find file: {0}\n".format(filetoconvert)) + return 1 + + with open(filetoconvert, 'x') as f: + with open(filecopy,'r') as c: + fileread = c.read() + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + #print ("DEBUG: %s "%converted) + print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + f.write(converted) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 1d9684363a4cc0efe5dadc3b18269c11a1f76a3b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 16 Oct 2018 08:50:30 +0100 Subject: [PATCH 004/589] Merge commit 'fadf5e954325f20d50142038be5aa428ad5d31af' --- compile/eclipsebuild/eclipseds5build.gsh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9fd5293..3061fea 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -154,7 +154,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } - doExport(exportfolder, exportpathlist, "", build_variant) + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." + } } catch (Exception e) { echo "Build failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') @@ -166,7 +169,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - doExport(exportfolder, exportpathlist, project) + result = doExport(exportfolder, exportpathlist, project) + if (result != 0) { + error "Failed to copy...." + } } } } From d202c89bfa3fc0e2ce83bf65cd7e0872246c53eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 10:07:49 +0100 Subject: [PATCH 005/589] Move json file comparison to subtree area --- differences/json.gsh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 differences/json.gsh diff --git a/differences/json.gsh b/differences/json.gsh new file mode 100644 index 0000000..47d9cc5 --- /dev/null +++ b/differences/json.gsh @@ -0,0 +1,15 @@ +def comparefiles(importedjsonfile, currentjsonfile) { + env.IMPORTED_JSON="${importedjsonfile}" + env.CURRENT_JSON="${currentjsonfile}" + if (isUnix()) { + sh """ + # jq -S sorts json output + diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + """ + } + else { + error "Windows not currently supporting json file comparison" + } +} + +return this; From 2393df8cb20b15b5a60ed6a6989263189df86005 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 13:32:04 +0100 Subject: [PATCH 006/589] Move meta artifact builder to common scripts subtree area --- artifacts/metaexport.gsh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 artifacts/metaexport.gsh diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.gsh new file mode 100644 index 0000000..368871e --- /dev/null +++ b/artifacts/metaexport.gsh @@ -0,0 +1,24 @@ + +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKSPACE}") { + artifactFolders = "" + for (subJob in subJobs) { + subJobObject = subJob[0] + artifactFolder = subJob[1] + if (subJobObject != null) { + subJobName = subJobObject.getProjectName() + subJobNumber = subJobObject.getNumber() + destFolder = "${exportRootFolder}/${artifactFolder}" + + dir("${destFolder}") { + deleteDir() + } + + echo "Copying artifacts from: Job: ${subJobName} Number: ${subJobNumber} to folder: ${destFolder}" + copyArtifacts fingerprintArtifacts: true, projectName: "${subJobName}", selector: specific("${subJobNumber}"), target: "${destFolder}" + artifactFolders += "${destFolder}/**," + } + } +} + +return this; From 18c1343663e761a9733f739c2bb6de4b3c783eea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 14:24:17 +0100 Subject: [PATCH 007/589] Move meta artifact builder to common scripts subtree area --- artifacts/metaexport.gsh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.gsh index 368871e..66a8153 100644 --- a/artifacts/metaexport.gsh +++ b/artifacts/metaexport.gsh @@ -1,6 +1,6 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKSPACE}") { +def combineSubjobArtifacts(subJobs) { artifactFolders = "" for (subJob in subJobs) { subJobObject = subJob[0] @@ -8,7 +8,7 @@ def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKS if (subJobObject != null) { subJobName = subJobObject.getProjectName() subJobNumber = subJobObject.getNumber() - destFolder = "${exportRootFolder}/${artifactFolder}" + destFolder = "${artifactFolder}" dir("${destFolder}") { deleteDir() @@ -19,6 +19,7 @@ def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKS artifactFolders += "${destFolder}/**," } } + return artifactFolders; } return this; From e7c44d4fd43da49cf3468a18a511757e61f3f650 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Oct 2018 00:09:04 +0100 Subject: [PATCH 008/589] Add utility to make build stamp representing a set of components --- versionnumbering/complexversioninfo.py | 220 +++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 versionnumbering/complexversioninfo.py diff --git a/versionnumbering/complexversioninfo.py b/versionnumbering/complexversioninfo.py new file mode 100644 index 0000000..be6ada4 --- /dev/null +++ b/versionnumbering/complexversioninfo.py @@ -0,0 +1,220 @@ +# encoding: utf-8 +''' +complexversionnumbering -- shortdesc + +complexversionnumbering generates a version folder with a common set of version information files in it for a complex component (consisting of many subcomponents) + +@author: dan + +@copyright: 2018 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2018-10-24' +__updated__ = '2018-10-24' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision, remoteinfo) + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return mastertypeversionprefix + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-p", "--stampprefix", dest="stampprefix", action="store", default="1.30000.0", help="A prefix to help identify this build [default: %(default)s]") + parser.add_argument("-s", "--subbuildstamp", dest="subbuildstamp", nargs='+', action="store", help="A subcomponent build stamp - there must be 1 or more") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + stampprefix = args.stampprefix + subbuildstamp = args.subbuildstamp + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Stamp prefix:", stampprefix, "sub build stamp:", subbuildstamp, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) From 1d959d0a85015f21525ec5ebaf656d5c118c8e11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Oct 2018 23:04:31 +0100 Subject: [PATCH 009/589] Detect branches with 'master' string better; Split numbers better in branch names --- versionnumbering/versioninfo.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index a9d8738..7e6043f 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -12,6 +12,7 @@ import os import sys import shutil import subprocess +import re from argparse import ArgumentParser from argparse import RawDescriptionHelpFormatter @@ -81,13 +82,18 @@ def getVersionPrefix(sourcebranch): sourcebranch = splits[-1] - if sourcebranch.lower().endswith("master"): # Special case + if sourcebranch.lower().find("master") != -1: # Special case return mastertypeversionprefix - splits = sourcebranch.split('.') + splits2 = re.sourcebranch(r'(\d+)', v) + splits = [] + for split in splits2: + if split.isdigit(): + splits.append(split) + versionprefix = "" if len(splits) < 1: - print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) versionprefix = defaultversionprefix else: splitnumbercount = 0 From 9a4601e41e9fe627df818126f6c241a91bf01b5e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Oct 2018 23:12:00 +0100 Subject: [PATCH 010/589] Detect branches with 'master' string better; Split numbers better in branch names --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 7e6043f..b0d8ba1 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -85,7 +85,7 @@ def getVersionPrefix(sourcebranch): if sourcebranch.lower().find("master") != -1: # Special case return mastertypeversionprefix - splits2 = re.sourcebranch(r'(\d+)', v) + splits2 = re.split(r'(\d+)', sourcebranch) splits = [] for split in splits2: if split.isdigit(): From 883ea65599072260c7d13e61c1cf792115e51d83 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 00:19:26 +0100 Subject: [PATCH 011/589] Add metaversion generation --- versionnumbering/complexversioninfo.py | 220 ------------------------- versionnumbering/metaversion.gsh | 41 +++++ 2 files changed, 41 insertions(+), 220 deletions(-) delete mode 100644 versionnumbering/complexversioninfo.py create mode 100644 versionnumbering/metaversion.gsh diff --git a/versionnumbering/complexversioninfo.py b/versionnumbering/complexversioninfo.py deleted file mode 100644 index be6ada4..0000000 --- a/versionnumbering/complexversioninfo.py +++ /dev/null @@ -1,220 +0,0 @@ -# encoding: utf-8 -''' -complexversionnumbering -- shortdesc - -complexversionnumbering generates a version folder with a common set of version information files in it for a complex component (consisting of many subcomponents) - -@author: dan - -@copyright: 2018 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2018-10-24' -__updated__ = '2018-10-24' - -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -slaveMachine = os.getenv("NODE_NAME", default="-") - -def resetFolder(versionfolder): - print("Removing folders and contents of: {0}".format(versionfolder)) - try: - shutil.rmtree(versionfolder) - except FileNotFoundError: - pass - - print("Creating folder: {0}".format(versionfolder)) - os.makedirs(versionfolder) - -def getGitRevision(gitrootpath, basenumber): - cwd = os.getcwd() - - path = os.path.expanduser(gitrootpath) - print("Changing current working directory to: {0}".format(path)) - os.chdir(path) - - print("Determining revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - changecount = completed.stdout.decode("utf-8").strip() - print("Change count: {0}".format(changecount)) - adjustedchangecount = int(changecount) + int(basenumber) - print("Adjusted change count: {0}".format(adjustedchangecount)) - - print("Determining current revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - revision = completed.stdout.decode("utf-8").strip() - print("Revision: {0}".format(revision)) - - print("Determining remote info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteinfo = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteinfo)) - - os.chdir(cwd) - print("Restored current working directory to: {0}".format(cwd)) - - return (adjustedchangecount, revision, remoteinfo) - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().endswith("master"): # Special case - return mastertypeversionprefix - - splits = sourcebranch.split('.') - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-p", "--stampprefix", dest="stampprefix", action="store", default="1.30000.0", help="A prefix to help identify this build [default: %(default)s]") - parser.add_argument("-s", "--subbuildstamp", dest="subbuildstamp", nargs='+', action="store", help="A subcomponent build stamp - there must be 1 or more") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") - parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - stampprefix = args.stampprefix - subbuildstamp = args.subbuildstamp - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber - - print("Stamp prefix:", stampprefix, "sub build stamp:", subbuildstamp, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) - - resetFolder(versionfolder) - - (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh new file mode 100644 index 0000000..b4b1ad7 --- /dev/null +++ b/versionnumbering/metaversion.gsh @@ -0,0 +1,41 @@ + +// Each version folder should have been created using version creation scripts in this Common_scripts folder +// as their are well-known files in each folder +def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = "${WORKSPACE}/metaversion") { + env.VLP = versionLabelPrefix + env.OF = outputFolder + totalrevision = 0 + if (isUnix()) { + sh """#!/bin/bash + + rm -rf "${OF}" + mkdir -p "${OF}" + if [ \$? -ne 0 ]; then exit 1; fi + + touch "${OF}/differences.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + else { + error "combineSubjobVersions script is not supported in Windows yet" + } + for (versionFolder in versionFolderList) { + env.differemcesFile = "${versionFolder}/differences.txt" + sh """#!/bin/bash + + echo ${differencesFile} >> "${OF}/differences.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + cat "${differencesFile}" >> "${OF}/differences.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ + revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + totalrevision += intrevision + echo "Accumulated revision is: ${totalrevision}" + } + writeFile file: "${OF}/metaversion.txt" text: versionLabelPrefix + "." + totalrevision +} + +return this; \ No newline at end of file From 55ef090a86d3f317b7a85189dc6cfca2808a7870 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 08:37:10 +0100 Subject: [PATCH 012/589] Add metaversion generation --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index b4b1ad7..c6eee74 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -35,7 +35,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = totalrevision += intrevision echo "Accumulated revision is: ${totalrevision}" } - writeFile file: "${OF}/metaversion.txt" text: versionLabelPrefix + "." + totalrevision + writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } return this; \ No newline at end of file From bca6d804dd82173d1e1bacc54ff0dee8ed7f62ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 09:13:34 +0100 Subject: [PATCH 013/589] Add metaversion generation --- versionnumbering/metaversion.gsh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index c6eee74..e1cdaf6 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -14,28 +14,46 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = touch "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi + + touch "${OF}/buildstamps.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ } else { error "combineSubjobVersions script is not supported in Windows yet" } for (versionFolder in versionFolderList) { - env.differemcesFile = "${versionFolder}/differences.txt" + env.differencesFile = "${versionFolder}/differences.txt" sh """#!/bin/bash - echo ${differencesFile} >> "${OF}/differences.txt" + echo ${differencesFile}: >> "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi cat "${differencesFile}" >> "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi """ + env.buildStampFile = "${versionFolder}/buildstamp.txt" + sh """#!/bin/bash + + echo ${versionFolder}: >> "${OF}/buildstamps.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + cat "${buildStampFile}" >> "${OF}/buildstamps.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" totalrevision += intrevision echo "Accumulated revision is: ${totalrevision}" } - writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision + if (versionLabelPrefix.endsWith(".")) { + writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + totalrevision + } + else { + writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision + } } return this; \ No newline at end of file From 614a4b1e6b47f71c967780af75faa9090fe0fa1a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Oct 2018 13:37:17 +0100 Subject: [PATCH 014/589] Add metaversion generation --- test.gsh | 7 ------- versionnumbering/metaversion.gsh | 13 +++++-------- 2 files changed, 5 insertions(+), 15 deletions(-) delete mode 100644 test.gsh diff --git a/test.gsh b/test.gsh deleted file mode 100644 index 23fcfc0..0000000 --- a/test.gsh +++ /dev/null @@ -1,7 +0,0 @@ -// Methods in this file will end up as object methods on the object that load returns. -def lookAtThis(String whoAreYou) { - echo "Look at this, ${whoAreYou}! You loaded this from another file!" -} - -return this; - diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index e1cdaf6..aab7319 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,15 +33,12 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = cat "${differencesFile}" >> "${OF}/differences.txt" if [ \$? -ne 0 ]; then exit 1; fi """ - env.buildStampFile = "${versionFolder}/buildstamp.txt" - sh """#!/bin/bash + buildStampFile = "${versionFolder}/buildstamp.txt" + buildStampFileContents = readFile "${buildStampFile}" + metabuildStampFile = "${OF}/buildstamps.txt" + metabuildStampFileContents = readFile "${metabuildStampFile}" + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ": " + buildStampFileContents - echo ${versionFolder}: >> "${OF}/buildstamps.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - cat "${buildStampFile}" >> "${OF}/buildstamps.txt" - if [ \$? -ne 0 ]; then exit 1; fi - """ revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" From 4d9723e6155fb264bcebfcd44e5ba79740c58f07 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 27 Oct 2018 10:25:22 +0100 Subject: [PATCH 015/589] MCS compile should use build stamp generated by Common_scripts --- versionnumbering/versioninfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index b0d8ba1..62af192 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -13,6 +13,7 @@ import sys import shutil import subprocess import re +import datetime from argparse import ArgumentParser from argparse import RawDescriptionHelpFormatter @@ -151,6 +152,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: f.write(slaveMachine) + with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: + f.write(str(datetime.datetime.now)) + def main(argv=None): # IGNORE:C0111 '''Command line options.''' From 34e273f84a839d485344e24a372defea78d50054 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 27 Oct 2018 10:45:59 +0100 Subject: [PATCH 016/589] Add MCS jira key search --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 4937134..2d3ee0d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -78,6 +78,9 @@ jiraprojectpattern = { 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), ), + 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), + ), 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), From 476b4a59489df06cc15a6c29b7bf4ef514ef4d41 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 27 Oct 2018 10:56:49 +0100 Subject: [PATCH 017/589] Change base build number to match previous build system --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 62af192..1a6a72b 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -153,7 +153,7 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou f.write(slaveMachine) with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now)) + f.write(str(datetime.datetime.now())) def main(argv=None): # IGNORE:C0111 '''Command line options.''' From a441a0ba43586b6ff9db5c90d146e59f4e135f3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Oct 2018 00:24:11 +0100 Subject: [PATCH 018/589] Handle case where there only one version folder --- versioncontrol/git.gsh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index dbc09ff..904394d 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -43,9 +43,13 @@ String getSHAFromVersion(String versionFolder) { boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def keys = versions.keySet() - if (keys.size() <= 1) { + if (keys.size() < 1) { return false - } else { + } + if (keys.size() == 1) { + return true + } + else { version = "" for (key in keys) { nextversion = versions.get(key) From eaa8de01d5169f93d19b98699bd066e1650b4aea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 11:43:56 +0000 Subject: [PATCH 019/589] Move autobuild scripts into their own folder --- configuration/configuration.gsh | 2 +- versioncontrol/git.gsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index b91e0a3..9f60c0a 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -gh = load("${WORKSPACE}/Common_scripts/versioncontrol/git.gsh") +gh = load("${COMMON_SCRIPTS}/versioncontrol/git.gsh") configurationCollection = [:] diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 904394d..51dbd5d 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,4 +1,4 @@ -clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") +clh = load("${COMMON_SCRIPTS}/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 29d0f5e2ad63ff325ff9d5b2972b5d4b7b59f640 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 11:52:23 +0000 Subject: [PATCH 020/589] Move autobuild scripts into their own folder --- configuration/configuration.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 9f60c0a..5688482 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -8,7 +8,7 @@ def registerConfigurationInfo(String configurationName, String configurationURL, } def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${WORKSPACE}")) + registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${AUTOBUILD_SCRIPTS}")) } def saveConfigurationInVersionInfo() { From 8f9f29cad5badfa19290a59e3d7e13a9a197214a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 16:24:16 +0000 Subject: [PATCH 021/589] When working out what version prefix to use from a revision - if there are no dots (but there are numbers) default to 1.25000.0. --- versionnumbering/versioninfo.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 1a6a72b..ae2b0ee 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -76,6 +76,7 @@ def getVersionPrefix(sourcebranch): defaultversionprefix = "1.30000.0." mastertypeversionprefix = "1.20000.0." + thirdpartyversionprefix = "1.25000.0." splits = sourcebranch.split('/') if len(splits) < 1: print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) @@ -88,9 +89,16 @@ def getVersionPrefix(sourcebranch): splits2 = re.split(r'(\d+)', sourcebranch) splits = [] + hasdot = False for split in splits2: if split.isdigit(): splits.append(split) + if split == ".": + hasdot = True + + if not hasdot: + print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) + return thirdpartyversionprefix versionprefix = "" if len(splits) < 1: From bc79cbfbee40fd48d6857947ee657067e05b68b3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 16:52:36 +0000 Subject: [PATCH 022/589] Prevent revision number going above 16 bits worth of integer --- versionnumbering/versioninfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index ae2b0ee..7122dba 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -25,6 +25,7 @@ __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") slaveMachine = os.getenv("NODE_NAME", default="-") +MAXVERSIONNUMBER = 65535 def resetFolder(versionfolder): print("Removing folders and contents of: {0}".format(versionfolder)) @@ -48,7 +49,7 @@ def getGitRevision(gitrootpath, basenumber): print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) - adjustedchangecount = int(changecount) + int(basenumber) + adjustedchangecount = (int(changecount) + int(basenumber) % (MAXVERSIONNUMBER + 1)) print("Adjusted change count: {0}".format(adjustedchangecount)) print("Determining current revision info using git command: {0}".format(gitExecutable)) From 2376df47584eeb5da65593a09c30758a0cd5c4cc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 17:05:52 +0000 Subject: [PATCH 023/589] Prevent revision number going above 16 bits worth of integer --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 7122dba..ceae653 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -49,7 +49,7 @@ def getGitRevision(gitrootpath, basenumber): print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) - adjustedchangecount = (int(changecount) + int(basenumber) % (MAXVERSIONNUMBER + 1)) + adjustedchangecount = int((int(changecount) + int(basenumber)) % (MAXVERSIONNUMBER + 1)) print("Adjusted change count: {0}".format(adjustedchangecount)) print("Determining current revision info using git command: {0}".format(gitExecutable)) From b38a59a7dc77c58fff619112425a299498e0ee7a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 18:40:28 +0000 Subject: [PATCH 024/589] Move some installer helper functionality to Common_scripts --- install/installhelper.gsh | 41 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 install/installhelper.gsh diff --git a/install/installhelper.gsh b/install/installhelper.gsh new file mode 100644 index 0000000..a55292d --- /dev/null +++ b/install/installhelper.gsh @@ -0,0 +1,41 @@ +// Given details about artifacts to import and details of installers to build, import artifacts that are required +def artifactsImporter(artifactDetails, installerDetails) { + artifactFetched = [:] + + for (installer in toBuild) { + def artifactKey = installerDetails.get(installer)[0] + //echo "To build: ${installer} use artifactKey: ${artifactKey}" + (sourceProject, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + env.importDest = destFolder + if (artifactFetched.get(artifactKey) == null) { + //echo "Artifact info: ${artifactInfo}" + copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" + def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" + def folderToCopy = "${destFolder}\\version" + echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" + fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) + //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) + artifactVersion = githelper.getSHAFromVersion("${destFolder}/version") + artifactFetched.put(artifactKey, artifactVersion) + } + } + return artifactFetched +} + +// Export any required binaries to the /bin folder +def exportBinaries(artifactDetails, installerDetails) { + for (installer in toBuild) { + (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + if (exportBinaries) { + (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + for (binaryFolder in binariesFolders) { + binariesSourceFolder = "${destinationFolder}/${binaryFolder}" + binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" + fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) + } + } + } +} + +return this; \ No newline at end of file From 28384f6ec3e8016f796f1e4fcbde28dce6d989ac Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 08:30:57 +0000 Subject: [PATCH 025/589] Only append differences to the metadifferences file if they exist --- versionnumbering/metaversion.gsh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index aab7319..9e09b34 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -24,15 +24,13 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = error "combineSubjobVersions script is not supported in Windows yet" } for (versionFolder in versionFolderList) { - env.differencesFile = "${versionFolder}/differences.txt" - sh """#!/bin/bash + differencesFile = "${versionFolder}/differences.txt" + if (fileExists("${differencesFile}")) { + currentContents = readFile file: "${OF}/differences.txt" + newContents = readFile file: "${differencesFile}" + writeFile file: "${OF}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + } - echo ${differencesFile}: >> "${OF}/differences.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - cat "${differencesFile}" >> "${OF}/differences.txt" - if [ \$? -ne 0 ]; then exit 1; fi - """ buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" metabuildStampFile = "${OF}/buildstamps.txt" From 53fd4d61ce4a19124c115aa017f984b0aa00cb0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 08:56:17 +0000 Subject: [PATCH 026/589] Add remote url to version information --- versionnumbering/versioninfo.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index ceae653..607ce96 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -64,10 +64,16 @@ def getGitRevision(gitrootpath, basenumber): remoteinfo = completed.stdout.decode("utf-8") print("Remote info: {0}".format(remoteinfo)) + print("Determining remote url using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "get-url", "--push", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteurl = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteurl)) + os.chdir(cwd) print("Restored current working directory to: {0}".format(cwd)) - return (adjustedchangecount, revision, remoteinfo) + return (adjustedchangecount, revision, remoteinfo, remoteurl) # Handle: # origin/master @@ -132,7 +138,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp): with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: f.write(buildjobname) @@ -148,6 +154,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: f.write(remoteinfo) + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: f.write(buildnumber) @@ -214,14 +223,14 @@ USAGE resetFolder(versionfolder) - (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) versionprefix = getVersionPrefix(sourcebranch) buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp) return 0 except Exception as e: From d842f1a87b9b3b2dd04002e1b0c9705a6d91c6ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 12:25:04 +0000 Subject: [PATCH 027/589] Output more git information in the meta version output --- versionnumbering/metaversion.gsh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 9e09b34..634da16 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,9 +33,13 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" + remoteurlFile = "${versionFolder}/remoteurl.txt" + remoteurlFileContents = readFile "${remoteurlFile}" + gitRevisionFile = "${versionFolder}/gitrevision.txt" + gitRevisionFileContents = readFile "${gitRevisionFile}" metabuildStampFile = "${OF}/buildstamps.txt" metabuildStampFileContents = readFile "${metabuildStampFile}" - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ": " + buildStampFileContents + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() From 469bf47a8cbbf070576bb2abf8e5d75ff4ee374c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 16:28:37 +0000 Subject: [PATCH 028/589] Handle the situation where a build does not provide a remoteurl.txt file --- versionnumbering/metaversion.gsh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 634da16..911e094 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,10 +33,18 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" + remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = readFile "${remoteurlFile}" + if (fileExists("${remoteurlFile}")) { + remoteurlFileContents = readFile "${remoteurlFile}" + } + else { + remoteurlFileContents = 'Unknown' + } + gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile "${gitRevisionFile}" + metabuildStampFile = "${OF}/buildstamps.txt" metabuildStampFileContents = readFile "${metabuildStampFile}" writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents From 75c7e069cccc27681245dfe06428a507cb9675be Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 20:28:30 +0000 Subject: [PATCH 029/589] Format metaversion output --- versionnumbering/metaversion.gsh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 911e094..d296fca 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -28,27 +28,27 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = if (fileExists("${differencesFile}")) { currentContents = readFile file: "${OF}/differences.txt" newContents = readFile file: "${differencesFile}" - writeFile file: "${OF}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + breakline = "***********************************************************************\n" + writeFile file: "${OF}/differences.txt", text: currentContents + breakline + "${differencesFile}: \n" + newContents } buildStampFile = "${versionFolder}/buildstamp.txt" - buildStampFileContents = readFile "${buildStampFile}" + buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" if (fileExists("${remoteurlFile}")) { - remoteurlFileContents = readFile "${remoteurlFile}" + remoteurlFileContents = readFile("${remoteurlFile}").trim() } else { remoteurlFileContents = 'Unknown' } gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile "${gitRevisionFile}" + gitRevisionFileContents = readFile("${gitRevisionFile}").trim() metabuildStampFile = "${OF}/buildstamps.txt" - metabuildStampFileContents = readFile "${metabuildStampFile}" + metabuildStampFileContents = readFile("${metabuildStampFile}") writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents - revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" From f6379b95dfd35b11455b6dd8b4432a58f68b1959 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 2 Nov 2018 17:36:15 +0000 Subject: [PATCH 030/589] Make up revision information from sub meta jobs --- versionnumbering/metaversion.gsh | 123 +++++++++++++++++++------------ 1 file changed, 75 insertions(+), 48 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index d296fca..cac6faf 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -1,66 +1,93 @@ +def mergemeta(versionFolder, outputFolder) { + differencesFile = "${versionFolder}/differences.txt" + currentContents = readFile file: "${outputFolder}/differences.txt" + newContents = readFile file: "${differencesFile}" + writeFile file: "${outputFolder}/differences.txt", text: currentContents + "\n" + newContents + + buildStampFile = "${versionFolder}/buildstamps.txt" + buildStampFileContents = readFile("${buildStampFile}").trim() + + metabuildStampFile = "${outputFolder}/buildstamps.txt" + metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\n" + "${buildStampFileContents}" + + if (fileExists("${versionFolder}/revision.txt")) { + revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + } + else { + intrevision = 1 + } + + return intrevision +} + +def appendcomponentinfo(versionFolder, outputFolder) { + differencesFile = "${versionFolder}/differences.txt" + if (fileExists("${differencesFile}")) { + currentContents = readFile file: "${outputFolder}/differences.txt" + newContents = readFile file: "${differencesFile}" + writeFile file: "${outputFolder}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + } + + buildStampFile = "${versionFolder}/buildstamp.txt" + buildStampFileContents = readFile("${buildStampFile}").trim() + + remoteurlFile = "${versionFolder}/remoteurl.txt" + if (fileExists("${remoteurlFile}")) { + remoteurlFileContents = readFile("${remoteurlFile}").trim() + } + else { + remoteurlFileContents = 'Unknown' + } + + gitRevisionFile = "${versionFolder}/gitrevision.txt" + gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + + metabuildStampFile = "${outputFolder}/buildstamps.txt" + if (fileExists("${differencesFile}")) { + metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" + } + else { + metabuildStampFileContents = "" + } + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + + revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + + return intrevision +} + // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = "${WORKSPACE}/metaversion") { - env.VLP = versionLabelPrefix - env.OF = outputFolder totalrevision = 0 - if (isUnix()) { - sh """#!/bin/bash - - rm -rf "${OF}" - mkdir -p "${OF}" - if [ \$? -ne 0 ]; then exit 1; fi - - touch "${OF}/differences.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - touch "${OF}/buildstamps.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - """ - } - else { - error "combineSubjobVersions script is not supported in Windows yet" + dir("${outputFolder}") { + deleteDir() } for (versionFolder in versionFolderList) { - differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - currentContents = readFile file: "${OF}/differences.txt" - newContents = readFile file: "${differencesFile}" - breakline = "***********************************************************************\n" - writeFile file: "${OF}/differences.txt", text: currentContents + breakline + "${differencesFile}: \n" + newContents - } - - buildStampFile = "${versionFolder}/buildstamp.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() - - remoteurlFile = "${versionFolder}/remoteurl.txt" - if (fileExists("${remoteurlFile}")) { - remoteurlFileContents = readFile("${remoteurlFile}").trim() + metaversionsignaturefile = "${versionFolder}/metaversion.txt" + // If the sub project is a meta job + if (fileExists("${metaversionsignaturefile}")) { + intrevision = mergemeta("${versionFolder}", outputFolder) } else { - remoteurlFileContents = 'Unknown' + intrevision = appendcomponentinfo("${versionFolder}", outputFolder) } - - gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - metabuildStampFile = "${OF}/buildstamps.txt" - metabuildStampFileContents = readFile("${metabuildStampFile}") - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents - revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - totalrevision += intrevision - echo "Accumulated revision is: ${totalrevision}" } + totalrevision += intrevision + echo "Accumulated revision is: ${totalrevision}" if (versionLabelPrefix.endsWith(".")) { - writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + totalrevision + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + totalrevision } else { - writeFile file: "${OF}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } + writeFile("${outputFolder}/revision.txt").trim(), text: totalrevision) } return this; \ No newline at end of file From 67895110b66406b57385835d80888ecfbd8d9c2a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 4 Nov 2018 17:34:35 +0000 Subject: [PATCH 031/589] Make up revision information from sub meta jobs --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index cac6faf..4a821be 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -87,7 +87,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = else { writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } - writeFile("${outputFolder}/revision.txt").trim(), text: totalrevision) + writeFile("${outputFolder}/revision.txt", text: totalrevision) } return this; \ No newline at end of file From dc3abfc7cf5e4823af8e22da5a83b28de8671721 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 4 Nov 2018 20:46:58 +0000 Subject: [PATCH 032/589] More debug; Common functions --- versionnumbering/metaversion.gsh | 43 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 4a821be..8849ae3 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -1,16 +1,26 @@ +def currentContents(file, newline = "\n", defaultContents = "") { + if (fileExists("${file}")) { + contents = readFile(file: "${file}").trim() + newline + } + else { + contents = defaultContents + } + return contents +} + def mergemeta(versionFolder, outputFolder) { + echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" + differencesFile = "${versionFolder}/differences.txt" - currentContents = readFile file: "${outputFolder}/differences.txt" newContents = readFile file: "${differencesFile}" - writeFile file: "${outputFolder}/differences.txt", text: currentContents + "\n" + newContents + metadifferencesFile = ${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents buildStampFile = "${versionFolder}/buildstamps.txt" buildStampFileContents = readFile("${buildStampFile}").trim() - metabuildStampFile = "${outputFolder}/buildstamps.txt" - metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\n" + "${buildStampFileContents}" + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents if (fileExists("${versionFolder}/revision.txt")) { revision = readFile("${versionFolder}/revision.txt").trim() @@ -25,36 +35,26 @@ def mergemeta(versionFolder, outputFolder) { } def appendcomponentinfo(versionFolder, outputFolder) { + echo "Appemding component version data from folder: ${versionFolder} to ${outputFolder}" + differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { - currentContents = readFile file: "${outputFolder}/differences.txt" newContents = readFile file: "${differencesFile}" - writeFile file: "${outputFolder}/differences.txt", text: currentContents + "${differencesFile}: \n" + newContents + metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" - if (fileExists("${remoteurlFile}")) { - remoteurlFileContents = readFile("${remoteurlFile}").trim() - } - else { - remoteurlFileContents = 'Unknown' - } + remoteurlFileContents = currentContents(remoteurlFile. "", 'Unknown') gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile("${gitRevisionFile}").trim() metabuildStampFile = "${outputFolder}/buildstamps.txt" - if (fileExists("${differencesFile}")) { - metabuildStampFileContents = readFile("${metabuildStampFile}") + "\r\n" - } - else { - metabuildStampFileContents = "" - } - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents - + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" @@ -70,6 +70,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = deleteDir() } for (versionFolder in versionFolderList) { + echo "Inspecting folder: ${versionFolder}" metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job if (fileExists("${metaversionsignaturefile}")) { From 3a5980aa9823dba95537bfd0b3e207266e5bd108 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 4 Nov 2018 23:20:10 +0000 Subject: [PATCH 033/589] More debug; Common functions --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 8849ae3..6e57d4c 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -14,7 +14,7 @@ def mergemeta(versionFolder, outputFolder) { differencesFile = "${versionFolder}/differences.txt" newContents = readFile file: "${differencesFile}" - metadifferencesFile = ${outputFolder}/differences.txt" + metadifferencesFile = "${outputFolder}/differences.txt" writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents buildStampFile = "${versionFolder}/buildstamps.txt" From 6c617020f10ae654e423f54a70790552123501ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 11:55:54 +0000 Subject: [PATCH 034/589] Accumulate the total revision --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 6e57d4c..61f8a3b 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -79,8 +79,8 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = else { intrevision = appendcomponentinfo("${versionFolder}", outputFolder) } + totalrevision += intrevision } - totalrevision += intrevision echo "Accumulated revision is: ${totalrevision}" if (versionLabelPrefix.endsWith(".")) { writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + totalrevision From 2f75cd2d14ef5a9836be3554a04fe20dcc6c16ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 14:32:04 +0000 Subject: [PATCH 035/589] Handle syntax error --- versionnumbering/metaversion.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 61f8a3b..22168ba 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -35,7 +35,7 @@ def mergemeta(versionFolder, outputFolder) { } def appendcomponentinfo(versionFolder, outputFolder) { - echo "Appemding component version data from folder: ${versionFolder} to ${outputFolder}" + echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { @@ -48,7 +48,7 @@ def appendcomponentinfo(versionFolder, outputFolder) { buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = currentContents(remoteurlFile. "", 'Unknown') + remoteurlFileContents = currentContents(remoteurlFile, "", 'Unknown').trim() gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile("${gitRevisionFile}").trim() From b6d7567c364668f072789725ef3b84523442f4bd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 15:06:41 +0000 Subject: [PATCH 036/589] Handle a release to git job name parameter - to be passed to metabuilder jobs --- versionnumbering/versioninfo.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 607ce96..370d9fc 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -138,7 +138,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp): +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: f.write(buildjobname) @@ -173,6 +173,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: f.write(str(datetime.datetime.now())) + with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: + f.write(releasejobname) + def main(argv=None): # IGNORE:C0111 '''Command line options.''' @@ -206,6 +209,7 @@ USAGE parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + parser.add_argument("-r", "--releasejobname", dest="releasejobname", action="store", default="-", help="release-to-git build job name [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -217,9 +221,10 @@ USAGE baseversionnumber = args.baseversionnumber buildjobname = args.buildjobname buildnumber = args.buildnumber + releasejobname = args.releasejobname print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber, "Release job name:", releasejobname) resetFolder(versionfolder) @@ -230,7 +235,7 @@ USAGE buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp) + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname) return 0 except Exception as e: From a6511a91beb44ab82ae292fd67fd9a96ddbfb3a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 15:47:49 +0000 Subject: [PATCH 037/589] Syntax error --- versionnumbering/metaversion.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 22168ba..084b604 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -88,7 +88,7 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = else { writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision } - writeFile("${outputFolder}/revision.txt", text: totalrevision) + writeFile(file: "${outputFolder}/revision.txt", text: totalrevision) } return this; \ No newline at end of file From 6fa63d47f3e796b00befcad58e8bdc76497462ef Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 16:19:52 +0000 Subject: [PATCH 038/589] Add release job name to metabuildstamp file --- versionnumbering/metaversion.gsh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 084b604..1dc19ca 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -53,8 +53,11 @@ def appendcomponentinfo(versionFolder, outputFolder) { gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + releaseJobNameFile = "${versionFolder}/releasejobname.txt" + releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() + metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" From dcdf1d3dc3a866d3cb73f3ce9f577105c6e89d3d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 5 Nov 2018 19:00:51 +0000 Subject: [PATCH 039/589] Correct integer printing --- versionnumbering/metaversion.gsh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 1dc19ca..9a50e2e 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -85,13 +85,8 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = totalrevision += intrevision } echo "Accumulated revision is: ${totalrevision}" - if (versionLabelPrefix.endsWith(".")) { - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + totalrevision - } - else { - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + "." + totalrevision - } - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision) + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() + writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) } return this; \ No newline at end of file From 0c2b471c3cea162ff6169768e2c9da8eabc4498d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 7 Nov 2018 22:36:03 +0000 Subject: [PATCH 040/589] Only build selected variants --- compile/eclipsebuild/eclipseds5build.gsh | 48 +++++++++++++++--------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 3061fea..078845e 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,14 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild. exportfolder = "${WORKSPACE}/bin") { + toBuild = [] + if (variantstobuild) { + toBuildUntrimmed = variantstobuild.split(',') + toBuildUntrimmed.each { + toBuild.add(it.trim()) + } + } // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { try { @@ -137,26 +144,31 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) try { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } } catch (Exception e) { echo "Build failed - Waiting for a while" From 9f1d3ca026847e066d3fe9921aa68a2b99812a80 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 7 Nov 2018 22:39:15 +0000 Subject: [PATCH 041/589] Only build selected variants --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 078845e..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,7 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild. exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}/bin") { toBuild = [] if (variantstobuild) { toBuildUntrimmed = variantstobuild.split(',') From 69de014d49076c0d25a24fe04600fedbc29e110f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:04:34 +0000 Subject: [PATCH 042/589] Condense autobuild startup boilerplate --- autobuilder.gsh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 autobuilder.gsh diff --git a/autobuilder.gsh b/autobuilder.gsh new file mode 100644 index 0000000..06a5123 --- /dev/null +++ b/autobuilder.gsh @@ -0,0 +1,39 @@ +// Assume that WORKSPACE environment variable is set + +def autobuildroot = "${WORKSPACE}/autobuild" +def commonscriptsroot = "${autobuildroot}/Common_scripts" + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + cleanWs() + unstash name: 'Clean workspace stash' + } + else { + echo "Not cleaning workspace" + } +} + +// Start up boilderplate +def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autobuildroot}/Common_scripts") { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") + configurationhelper.registerPipelineScriptConfiguration() + +def commonscripts() + return commonscriptsroot +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.gsh") +} + +return this; \ No newline at end of file From 3e7191297956761eead46374cc6cfd7348a97136 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:08:35 +0000 Subject: [PATCH 043/589] Condense autobuild startup boilerplate --- autobuilder.gsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 06a5123..7a7efa1 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -27,8 +27,9 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") configurationhelper.registerPipelineScriptConfiguration() +} -def commonscripts() +def commonscripts() { return commonscriptsroot } From 12f680285f37934d9dc4ff56b7b961149febbc2f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:13:21 +0000 Subject: [PATCH 044/589] Condense autobuild startup boilerplate --- autobuilder.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 7a7efa1..1657815 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,7 +1,7 @@ // Assume that WORKSPACE environment variable is set -def autobuildroot = "${WORKSPACE}/autobuild" -def commonscriptsroot = "${autobuildroot}/Common_scripts" +String autobuildroot = "${WORKSPACE}/autobuild" +String commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From a870eb3e57496a1ff78e182fc9cb543d2f50bfd0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:23:19 +0000 Subject: [PATCH 045/589] Condense autobuild startup boilerplate --- autobuilder.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 1657815..88b68a0 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,7 +1,10 @@ // Assume that WORKSPACE environment variable is set -String autobuildroot = "${WORKSPACE}/autobuild" -String commonscriptsroot = "${autobuildroot}/Common_scripts" +@Field +def autobuildroot = "${WORKSPACE}/autobuild" + +@Field +def commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 16c6e92ac9beaa917a8acde5802a491eba34b27e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:25:18 +0000 Subject: [PATCH 046/589] Condense autobuild startup boilerplate --- autobuilder.gsh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 88b68a0..bf42e4c 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,10 +1,8 @@ // Assume that WORKSPACE environment variable is set -@Field -def autobuildroot = "${WORKSPACE}/autobuild" +autobuildroot = "${WORKSPACE}/autobuild" -@Field -def commonscriptsroot = "${autobuildroot}/Common_scripts" +commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 9403f706aba7c1147d2ddce2a119edd6bae9b54b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:31:25 +0000 Subject: [PATCH 047/589] Condense autobuild startup boilerplate --- configuration/configuration.gsh | 2 +- versioncontrol/git.gsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 5688482..dbaf921 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -gh = load("${COMMON_SCRIPTS}/versioncontrol/git.gsh") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.gsh") configurationCollection = [:] diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 51dbd5d..1462a8e 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,4 +1,4 @@ -clh = load("${COMMON_SCRIPTS}/commandline/commandline.gsh") +clh = load("${autobuilder.commonscripts()}/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 3f3ee1c70997abd6693ea0c5224bbb8ec99f57bf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:37:04 +0000 Subject: [PATCH 048/589] Condense autobuild startup boilerplate --- autobuilder.gsh | 4 ++++ configuration/configuration.gsh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index bf42e4c..6ec0e38 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -34,6 +34,10 @@ def commonscripts() { return commonscriptsroot } +def getautobuildpath() { + return autobuildroot +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.gsh") } diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index dbaf921..f131c98 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -8,7 +8,7 @@ def registerConfigurationInfo(String configurationName, String configurationURL, } def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${AUTOBUILD_SCRIPTS}")) + registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } def saveConfigurationInVersionInfo() { From 94941c199f983f328906b85c02394fc77fb42326 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:46:24 +0000 Subject: [PATCH 049/589] Change .gsh suffix to .groovy --- artifacts/{metaexport.gsh => metaexport.groovy} | 0 autobuilder.gsh => autobuilder.groovy | 0 commandline/{commandline.gsh => commandline.groovy} | 0 .../{disableparallelbuilds.gsh => disableparallelbuilds.groovy} | 0 .../eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} | 0 configuration/{configuration.gsh => configuration.groovy} | 0 differences/{json.gsh => json.groovy} | 0 install/{installhelper.gsh => installhelper.groovy} | 0 versioncontrol/{git.gsh => git.groovy} | 0 versionnumbering/{metaversion.gsh => metaversion.groovy} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename artifacts/{metaexport.gsh => metaexport.groovy} (100%) rename autobuilder.gsh => autobuilder.groovy (100%) rename commandline/{commandline.gsh => commandline.groovy} (100%) rename compile/eclipsebuild/{disableparallelbuilds.gsh => disableparallelbuilds.groovy} (100%) rename compile/eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} (100%) rename configuration/{configuration.gsh => configuration.groovy} (100%) rename differences/{json.gsh => json.groovy} (100%) rename install/{installhelper.gsh => installhelper.groovy} (100%) rename versioncontrol/{git.gsh => git.groovy} (100%) rename versionnumbering/{metaversion.gsh => metaversion.groovy} (100%) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.groovy similarity index 100% rename from artifacts/metaexport.gsh rename to artifacts/metaexport.groovy diff --git a/autobuilder.gsh b/autobuilder.groovy similarity index 100% rename from autobuilder.gsh rename to autobuilder.groovy diff --git a/commandline/commandline.gsh b/commandline/commandline.groovy similarity index 100% rename from commandline/commandline.gsh rename to commandline/commandline.groovy diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.groovy similarity index 100% rename from compile/eclipsebuild/disableparallelbuilds.gsh rename to compile/eclipsebuild/disableparallelbuilds.groovy diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.groovy similarity index 100% rename from compile/eclipsebuild/eclipseds5build.gsh rename to compile/eclipsebuild/eclipseds5build.groovy diff --git a/configuration/configuration.gsh b/configuration/configuration.groovy similarity index 100% rename from configuration/configuration.gsh rename to configuration/configuration.groovy diff --git a/differences/json.gsh b/differences/json.groovy similarity index 100% rename from differences/json.gsh rename to differences/json.groovy diff --git a/install/installhelper.gsh b/install/installhelper.groovy similarity index 100% rename from install/installhelper.gsh rename to install/installhelper.groovy diff --git a/versioncontrol/git.gsh b/versioncontrol/git.groovy similarity index 100% rename from versioncontrol/git.gsh rename to versioncontrol/git.groovy diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.groovy similarity index 100% rename from versionnumbering/metaversion.gsh rename to versionnumbering/metaversion.groovy From 76913ba017012d451f6681d6bf44704178515ae9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:54:02 +0000 Subject: [PATCH 050/589] Change .gsh suffix to .groovy --- autobuilder.groovy | 4 ++-- configuration/configuration.groovy | 2 +- versioncontrol/git.groovy | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 6ec0e38..54cfb12 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob conditionalclean() - configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") + configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") configurationhelper.registerPipelineScriptConfiguration() } @@ -39,7 +39,7 @@ def getautobuildpath() { } def configurationhelper() { - return load("${commonscripts()}/configuration/configuration.gsh") + return load("${commonscripts()}/configuration/configuration.groovy") } return this; \ No newline at end of file diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index f131c98..1f0c095 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,4 +1,4 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.gsh") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") configurationCollection = [:] diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1462a8e..031094b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,4 +1,4 @@ -clh = load("${autobuilder.commonscripts()}/commandline/commandline.gsh") +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From ad68c4caf5f990cca4a774d46dac162666d1817d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 22:35:36 +0000 Subject: [PATCH 051/589] Remove clean boilerplate --- clean/clean.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 clean/clean.groovy diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file From e3533bd5f140a1561520b416ef585f366ed9dca4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 23:29:30 +0000 Subject: [PATCH 052/589] Remove versionnumbering boilerplate --- versionnumbering/generate.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 versionnumbering/generate.groovy diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..6e87392 --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,20 @@ + +def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${WORKSPACE}/version", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { + cs = autobuilder.commonscripts() + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${cs}%\\versionnumbering\\versioninfo.py" -s "%${sourcerevision}%" -v "${versionfolder}" -g "${sourcepath}" -b %${basebuildnumber}% -j "%${jobname}%" -n "%${buildid}%" -r "%${releasejobname}%" + """ + } + env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() + env.BUILD_STAMP_COMMA_SEP = readFile("${WORKSPACE}/version/buildstampcommasep.txt").trim() + env.SRC_REVISION = readFile("${WORKSPACE}/version/revision.txt").trim() + configurationhelper.saveConfigurationInVersionInfo(); +} + +return this; \ No newline at end of file From b0077bb1df56c27f18921cd7184d6e939fa43281 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 23:49:04 +0000 Subject: [PATCH 053/589] Remove differences boilerplate --- differences/do.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 differences/do.groovy diff --git a/differences/do.groovy b/differences/do.groovy new file mode 100644 index 0000000..9dc1807 --- /dev/null +++ b/differences/do.groovy @@ -0,0 +1,17 @@ + +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { + cs = autobuilder.commonscripts() + + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${cs}%\\differences\\show_release_differences.py" -g "%${sourcepath}%" -s "%${sourcerevision}%" -p "%${previoussourcerevision}%" -j ${jirakey} -o "opposite notcommon" -v "%${versionfolder}%" + """ + } +} + +return this; From 1d7346e5a02c0018986e56a47987adf197874731 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 11 Nov 2018 11:43:28 +0000 Subject: [PATCH 054/589] Remove boilerplate --- versionnumbering/do.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 versionnumbering/do.groovy diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..9b4a42e --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,20 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + filetoconvert = convertdetail[0] + pattern = convertdetail[1] + replacement = convertdetail[2] + if (isUnix()) { + sh """ + "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + """ + } + } +} + +return this; From b92a8261897b30807c06198982d92a5db1cfd805 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 12 Nov 2018 10:50:37 +0000 Subject: [PATCH 055/589] Rename .gsh files to .groovy --- artifacts/{metaexport.gsh => metaexport.groovy} | 0 commandline/{commandline.gsh => commandline.groovy} | 0 .../{disableparallelbuilds.gsh => disableparallelbuilds.groovy} | 0 .../eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} | 0 configuration/{configuration.gsh => configuration.groovy} | 0 differences/{json.gsh => json.groovy} | 0 install/{installhelper.gsh => installhelper.groovy} | 0 versioncontrol/{git.gsh => git.groovy} | 0 versionnumbering/{metaversion.gsh => metaversion.groovy} | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename artifacts/{metaexport.gsh => metaexport.groovy} (100%) rename commandline/{commandline.gsh => commandline.groovy} (100%) rename compile/eclipsebuild/{disableparallelbuilds.gsh => disableparallelbuilds.groovy} (100%) rename compile/eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} (100%) rename configuration/{configuration.gsh => configuration.groovy} (100%) rename differences/{json.gsh => json.groovy} (100%) rename install/{installhelper.gsh => installhelper.groovy} (100%) rename versioncontrol/{git.gsh => git.groovy} (100%) rename versionnumbering/{metaversion.gsh => metaversion.groovy} (100%) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.groovy similarity index 100% rename from artifacts/metaexport.gsh rename to artifacts/metaexport.groovy diff --git a/commandline/commandline.gsh b/commandline/commandline.groovy similarity index 100% rename from commandline/commandline.gsh rename to commandline/commandline.groovy diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.groovy similarity index 100% rename from compile/eclipsebuild/disableparallelbuilds.gsh rename to compile/eclipsebuild/disableparallelbuilds.groovy diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.groovy similarity index 100% rename from compile/eclipsebuild/eclipseds5build.gsh rename to compile/eclipsebuild/eclipseds5build.groovy diff --git a/configuration/configuration.gsh b/configuration/configuration.groovy similarity index 100% rename from configuration/configuration.gsh rename to configuration/configuration.groovy diff --git a/differences/json.gsh b/differences/json.groovy similarity index 100% rename from differences/json.gsh rename to differences/json.groovy diff --git a/install/installhelper.gsh b/install/installhelper.groovy similarity index 100% rename from install/installhelper.gsh rename to install/installhelper.groovy diff --git a/versioncontrol/git.gsh b/versioncontrol/git.groovy similarity index 100% rename from versioncontrol/git.gsh rename to versioncontrol/git.groovy diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.groovy similarity index 100% rename from versionnumbering/metaversion.gsh rename to versionnumbering/metaversion.groovy From 4f62592e7f7bf0c91cb372f12d10c83fa208d4c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 13 Nov 2018 09:39:25 +0000 Subject: [PATCH 056/589] buildstamp.txt -> buildstamps.txt --- versionnumbering/metaversion.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 9a50e2e..e566aa1 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -44,7 +44,7 @@ def appendcomponentinfo(versionFolder, outputFolder) { writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } - buildStampFile = "${versionFolder}/buildstamp.txt" + buildStampFile = "${versionFolder}/buildstamps.txt" buildStampFileContents = readFile("${buildStampFile}").trim() remoteurlFile = "${versionFolder}/remoteurl.txt" From 2b616e5397f88d7acbbdca593db7a7ce9ecae45f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:21:52 +0000 Subject: [PATCH 057/589] Handle version numbering in windows --- versionnumbering/generate.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 6e87392..cf02860 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -8,7 +8,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } else { bat """ - "%${PYTHON}%" -u "%${cs}%\\versionnumbering\\versioninfo.py" -s "%${sourcerevision}%" -v "${versionfolder}" -g "${sourcepath}" -b %${basebuildnumber}% -j "%${jobname}%" -n "%${buildid}%" -r "%${releasejobname}%" + \"${PYTHON}\" -u \"${cs}\\versionnumbering\\versioninfo.py\" -s \"${sourcerevision}\" -v \"${versionfolder}\" -g \"${sourcepath}\" -b ${basebuildnumber} -j \"${jobname}\" -n \"${buildid}\" -r \"${releasejobname}\" """ } env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() @@ -17,4 +17,4 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam configurationhelper.saveConfigurationInVersionInfo(); } -return this; \ No newline at end of file +return this; From 8ce03b4e10a7016e46bfd9f1f91c30822fdc89ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:28:10 +0000 Subject: [PATCH 058/589] Handle version numbering in windows --- differences/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/do.groovy b/differences/do.groovy index 9dc1807..7279c0b 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -9,7 +9,7 @@ def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, ver } else { bat """ - "%${PYTHON}%" -u "%${cs}%\\differences\\show_release_differences.py" -g "%${sourcepath}%" -s "%${sourcerevision}%" -p "%${previoussourcerevision}%" -j ${jirakey} -o "opposite notcommon" -v "%${versionfolder}%" + \"${PYTHON}\" -u \"${cs}\\differences\\show_release_differences.py\" -g \"${sourcepath}\" -s \"${sourcerevision}\" -p \"${previoussourcerevision}\" -j ${jirakey} -o \"opposite notcommon\" -v \"${versionfolder}\" """ } } From 143670fbc6aba03871871b44aae0e7cd6d5b6ffe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:44:11 +0000 Subject: [PATCH 059/589] Handle version numbering in windows --- differences/do.groovy | 9 +++------ versionnumbering/generate.groovy | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 7279c0b..d0a489b 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,15 +2,12 @@ def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() + diffcmd = '"${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}"' if (isUnix()) { - sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" - """ + sh "${diffcmd}" } else { - bat """ - \"${PYTHON}\" -u \"${cs}\\differences\\show_release_differences.py\" -g \"${sourcepath}\" -s \"${sourcerevision}\" -p \"${previoussourcerevision}\" -j ${jirakey} -o \"opposite notcommon\" -v \"${versionfolder}\" - """ + bat "${diffcmd}" } } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cf02860..2776130 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -8,7 +8,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } else { bat """ - \"${PYTHON}\" -u \"${cs}\\versionnumbering\\versioninfo.py\" -s \"${sourcerevision}\" -v \"${versionfolder}\" -g \"${sourcepath}\" -b ${basebuildnumber} -j \"${jobname}\" -n \"${buildid}\" -r \"${releasejobname}\" + "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" """ } env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() From 4c737131d55369f5deb9d2919c9632b401bea703 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 14 Nov 2018 12:45:05 +0000 Subject: [PATCH 060/589] Handle version numbering in windows --- differences/do.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index d0a489b..b3039a9 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,12 +2,15 @@ def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() - diffcmd = '"${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}"' if (isUnix()) { - sh "${diffcmd}" + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ } else { - bat "${diffcmd}" + bat """ + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ } } From bf1cba0f8baccaa4c5ea3bbe8ee60f870784a4ba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 15 Nov 2018 23:53:08 +0000 Subject: [PATCH 061/589] Prefix each line of a metaversion file with path to relevant artifact --- versionnumbering/metaversion.groovy | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index e566aa1..00256cc 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -9,7 +9,16 @@ def currentContents(file, newline = "\n", defaultContents = "") { return contents } -def mergemeta(versionFolder, outputFolder) { +def prefixline(rootpath, filepath) { + fileContents = readFile("${filepath}").trim() + newFileContents = "" + fileContents.eachline { + newFileContents += "${rootpath}" + "/" + it + } + return newFileContents +} + +def mergemeta(rootpath, versionFolder, outputFolder) { echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" differencesFile = "${versionFolder}/differences.txt" @@ -18,7 +27,7 @@ def mergemeta(versionFolder, outputFolder) { writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() + buildStampFileContents = prefixline(rootpath, "${buildStampFile}") metabuildStampFile = "${outputFolder}/buildstamps.txt" writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents @@ -34,7 +43,7 @@ def mergemeta(versionFolder, outputFolder) { return intrevision } -def appendcomponentinfo(versionFolder, outputFolder) { +def appendcomponentinfo(rootpath, versionFolder, outputFolder) { echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" differencesFile = "${versionFolder}/differences.txt" @@ -57,7 +66,7 @@ def appendcomponentinfo(versionFolder, outputFolder) { releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents + writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" @@ -67,20 +76,21 @@ def appendcomponentinfo(versionFolder, outputFolder) { // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = "${WORKSPACE}/metaversion") { +def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { totalrevision = 0 dir("${outputFolder}") { deleteDir() } - for (versionFolder in versionFolderList) { + for (artifactItem in artifactList) { + (rootpath, versionFolder) = artifactItem echo "Inspecting folder: ${versionFolder}" metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta("${versionFolder}", outputFolder) + intrevision = mergemeta(rootpath, versionFolder, outputFolder) } else { - intrevision = appendcomponentinfo("${versionFolder}", outputFolder) + intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) } totalrevision += intrevision } From 7114a1bf1c98f95d8d51b3c76176585a21d594d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 13:43:05 +0000 Subject: [PATCH 062/589] Remove some scope of variables being created --- autobuilder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 54cfb12..eb32dfc 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,8 +1,8 @@ // Assume that WORKSPACE environment variable is set -autobuildroot = "${WORKSPACE}/autobuild" +def autobuildroot = "${WORKSPACE}/autobuild" -commonscriptsroot = "${autobuildroot}/Common_scripts" +def commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { @@ -26,8 +26,8 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob conditionalclean() - configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") - configurationhelper.registerPipelineScriptConfiguration() + def ch = load("${commonscriptsroot}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration() } def commonscripts() { From 1552da9e07db5be1e461548d349d6844a81e92dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 13:50:31 +0000 Subject: [PATCH 063/589] Remove some scope of variables being created --- autobuilder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index eb32dfc..f07e945 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -1,8 +1,8 @@ // Assume that WORKSPACE environment variable is set -def autobuildroot = "${WORKSPACE}/autobuild" +autobuildroot = "${WORKSPACE}/autobuild" -def commonscriptsroot = "${autobuildroot}/Common_scripts" +commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 09a1c4fa507de3b417a2612fe80d5ea9cedc94f8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 14:04:56 +0000 Subject: [PATCH 064/589] Locate required githelper script --- install/installhelper.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index a55292d..f523799 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -15,7 +15,8 @@ def artifactsImporter(artifactDetails, installerDetails) { echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) - artifactVersion = githelper.getSHAFromVersion("${destFolder}/version") + def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + artifactVersion = gh.getSHAFromVersion("${destFolder}/version") artifactFetched.put(artifactKey, artifactVersion) } } From fdff4d689313b5cf358ca33db588cb2fcbe316c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 16 Nov 2018 23:29:39 +0000 Subject: [PATCH 065/589] Correct file line handling --- versionnumbering/metaversion.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 00256cc..3b8d7c1 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -10,10 +10,13 @@ def currentContents(file, newline = "\n", defaultContents = "") { } def prefixline(rootpath, filepath) { - fileContents = readFile("${filepath}").trim() - newFileContents = "" - fileContents.eachline { - newFileContents += "${rootpath}" + "/" + it + def fileContents = readFile("${filepath}").trim() + def fileContentsNormalized = fileContents.normalize() + + def newFileContents = "" + def lines = fileContentsNormalized.readLines() + for (aline in lines) { + newFileContents += "${rootpath}" + "/" + aline } return newFileContents } From 5ee5219418ae4de60c76db138ac8b912365af0b9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 11:36:47 +0000 Subject: [PATCH 066/589] Add platform specific line endings definition --- autobuilder.groovy | 4 +++- platform/lineendings.groovy | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 platform/lineendings.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 54cfb12..ef4f938 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,6 +28,8 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") configurationhelper.registerPipelineScriptConfiguration() + + lineendings = load("${commonscriptsroot}/platform/lineendings.groovy") } def commonscripts() { @@ -42,4 +44,4 @@ def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } -return this; \ No newline at end of file +return this; diff --git a/platform/lineendings.groovy b/platform/lineendings.groovy new file mode 100644 index 0000000..9657b0d --- /dev/null +++ b/platform/lineendings.groovy @@ -0,0 +1,5 @@ + +// Put ${nl} into strings when wanting a newline +nl = System.properties['line.separator'] + +return this; From ce133e9ba9ca4812c0afa08a6c5fb60aef42f3ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 11:55:39 +0000 Subject: [PATCH 067/589] Put line endings on metaversion file --- versionnumbering/metaversion.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 3b8d7c1..1f6cff6 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -1,4 +1,6 @@ +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") + def currentContents(file, newline = "\n", defaultContents = "") { if (fileExists("${file}")) { contents = readFile(file: "${file}").trim() + newline @@ -16,7 +18,7 @@ def prefixline(rootpath, filepath) { def newFileContents = "" def lines = fileContentsNormalized.readLines() for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + newFileContents += "${rootpath}" + "/" + aline + "${nl}" } return newFileContents } From 18cc8e0609829df0468474e9c2e70285d6ec12d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 20:25:15 +0000 Subject: [PATCH 068/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 31 ++++++++++++++++ file/utils.groovy | 16 ++++++++ publishing/utils.groovy | 25 +++++++++++++ versionnumbering/generate.groovy | 11 ++++-- versionnumbering/metaversion.groovy | 57 +++++++++++++++-------------- 5 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 checksum/utils.groovy create mode 100644 file/utils.groovy create mode 100644 publishing/utils.groovy diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..b1a714f --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,31 @@ + +env.CHECKSUMARTIFACTS = "checksums/**" +def CHECKSUMFOLDER = "${WORKSPACE}/checksums" + +// SHA256SUM environment variable must be set to sha256sum utility +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(checksummable) { + if !fileExists("${SHA256SUM}") { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + } + sh """ + rm -rf "${CHECKSUMFOLDER}" + mkdir -p "${CHECKSUMFOLDER}" + """ + for (filepath in checksummable) { + def files = findFiles(glob: filepath) + for (file in files) { + if isUnix() { + sh """#!/bin/bash + "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + """ + } + else { + bat """ + "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + """ + } + } + } +} + diff --git a/file/utils.groovy b/file/utils.groovy new file mode 100644 index 0000000..0ffa47c --- /dev/null +++ b/file/utils.groovy @@ -0,0 +1,16 @@ + +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") + +// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents +def currentContents(file, newline = "${nl}", defaultContents = "") { + def contents = "" + if (fileExists("${file}")) { + contents = readFile(file: "${file}").trim() + newline + } + else { + contents = defaultContents + } + return contents +} + +return this; diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..a10fd76 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,25 @@ + +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") + +env.PUBLISHABLEARTIFACTS = "publishable/**" +def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" +def PUBLISHABLEFILE = "${WORKSPACE}/publishable/list.json" + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(publishable) { + sh """ + rm -rf "${PUBLISHABLEFOLDER}" + mkdir -p "${PUBLISHABLEFOLDER}" + """ + filelist = [] + for (filepath in publishable) { + def files = findFiles(glob: filepath) + for (file in files) { + if (!file.directory) { + filelist.add(file.path) + } + } + } + def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) + writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) +} \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 2776130..6f9f803 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,5 +1,8 @@ -def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${WORKSPACE}/version", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { +env.VERSIONARTIFACTS = "version/**" +def VERSIONFOLDER = "${WORKSPACE}/version" + +def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { cs = autobuilder.commonscripts() if (isUnix()) { sh """ @@ -11,9 +14,9 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" """ } - env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() - env.BUILD_STAMP_COMMA_SEP = readFile("${WORKSPACE}/version/buildstampcommasep.txt").trim() - env.SRC_REVISION = readFile("${WORKSPACE}/version/revision.txt").trim() + env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() + env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() + env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() configurationhelper.saveConfigurationInVersionInfo(); } diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 1f6cff6..ff20a86 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -1,5 +1,6 @@ def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") def currentContents(file, newline = "\n", defaultContents = "") { if (fileExists("${file}")) { @@ -26,18 +27,19 @@ def prefixline(rootpath, filepath) { def mergemeta(rootpath, versionFolder, outputFolder) { echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - differencesFile = "${versionFolder}/differences.txt" - newContents = readFile file: "${differencesFile}" - metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents + def differencesFile = "${versionFolder}/differences.txt" + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents + def intrevision = 0 if (fileExists("${versionFolder}/revision.txt")) { - revision = readFile("${versionFolder}/revision.txt").trim() + def revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" } @@ -51,29 +53,29 @@ def mergemeta(rootpath, versionFolder, outputFolder) { def appendcomponentinfo(rootpath, versionFolder, outputFolder) { echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - differencesFile = "${versionFolder}/differences.txt" + def differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { - newContents = readFile file: "${differencesFile}" - metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } - buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = readFile("${buildStampFile}").trim() - remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = currentContents(remoteurlFile, "", 'Unknown').trim() + def remoteurlFile = "${versionFolder}/remoteurl.txt" + def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + def gitRevisionFile = "${versionFolder}/gitrevision.txt" + def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - releaseJobNameFile = "${versionFolder}/releasejobname.txt" - releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() + def releaseJobNameFile = "${versionFolder}/releasejobname.txt" + def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents + def revision = readFile("${versionFolder}/revision.txt").trim() + def intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" return intrevision @@ -82,15 +84,16 @@ def appendcomponentinfo(rootpath, versionFolder, outputFolder) { // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - totalrevision = 0 + def totalrevision = 0 dir("${outputFolder}") { deleteDir() } for (artifactItem in artifactList) { (rootpath, versionFolder) = artifactItem echo "Inspecting folder: ${versionFolder}" - metaversionsignaturefile = "${versionFolder}/metaversion.txt" + def metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job + def intrevision = 0 if (fileExists("${metaversionsignaturefile}")) { intrevision = mergemeta(rootpath, versionFolder, outputFolder) } From 583cffb0969e1b58848d252b55403b7a47e784d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 20:35:46 +0000 Subject: [PATCH 069/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 6 ++---- versionnumbering/generate.groovy | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index b1a714f..a961dc8 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,6 +1,5 @@ env.CHECKSUMARTIFACTS = "checksums/**" -def CHECKSUMFOLDER = "${WORKSPACE}/checksums" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace @@ -8,6 +7,7 @@ def generatelist(checksummable) { if !fileExists("${SHA256SUM}") { error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } + def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ rm -rf "${CHECKSUMFOLDER}" mkdir -p "${CHECKSUMFOLDER}" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index a10fd76..2fdbb73 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,12 +1,9 @@ -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - env.PUBLISHABLEARTIFACTS = "publishable/**" -def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" -def PUBLISHABLEFILE = "${WORKSPACE}/publishable/list.json" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { + def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" sh """ rm -rf "${PUBLISHABLEFOLDER}" mkdir -p "${PUBLISHABLEFOLDER}" @@ -21,5 +18,6 @@ def generatelist(publishable) { } } def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) + def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) } \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 6f9f803..cb5e944 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,6 +1,6 @@ env.VERSIONARTIFACTS = "version/**" -def VERSIONFOLDER = "${WORKSPACE}/version" +VERSIONFOLDER = "${WORKSPACE}/version" def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { cs = autobuilder.commonscripts() From b7433f290fa8a05272a6973fd53b97b5896c584f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 21:29:07 +0000 Subject: [PATCH 070/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index a961dc8..e4c37ca 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,8 +4,20 @@ env.CHECKSUMARTIFACTS = "checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - if !fileExists("${SHA256SUM}") { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + def sha256sumfound = true + if (!fileExists("${SHA256SUM}")) { + def sha256sumfound = false + if (isUnix()) { + def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() + if (sha256sumpath) { + echo "Found sha256sum executable here: + env.SHA256SUM = sha256sumpath + sha256sumfound = true + } + } + if (!sha256sumfound) { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + } } def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ From 660003f7b49f43fb3f26a53255aab1382b2cbf42 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 21:50:35 +0000 Subject: [PATCH 071/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index e4c37ca..63c96eb 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -10,7 +10,7 @@ def generatelist(checksummable) { if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { - echo "Found sha256sum executable here: + echo "Found sha256sum executable here: ${sha256sumpath}" env.SHA256SUM = sha256sumpath sha256sumfound = true } From 53c2f1110619cf4d49ae0a65f91b4427744de9a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:20:30 +0000 Subject: [PATCH 072/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 63c96eb..0d76756 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -27,7 +27,7 @@ def generatelist(checksummable) { for (filepath in checksummable) { def files = findFiles(glob: filepath) for (file in files) { - if isUnix() { + if (isUnix()) { sh """#!/bin/bash "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ From 7c55b96a55782302e1fd92b3b10b0532a9d1b04b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:24:30 +0000 Subject: [PATCH 073/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 0d76756..e358b94 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -6,7 +6,7 @@ env.CHECKSUMARTIFACTS = "checksums/**" def generatelist(checksummable) { def sha256sumfound = true if (!fileExists("${SHA256SUM}")) { - def sha256sumfound = false + sha256sumfound = false if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { From dcdf2c5fd171c8c697d06d9dfedc854b33107ad9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:39:48 +0000 Subject: [PATCH 074/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index e358b94..a46cd43 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -25,16 +25,18 @@ def generatelist(checksummable) { mkdir -p "${CHECKSUMFOLDER}" """ for (filepath in checksummable) { + echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { + echo "Checksumming ${file.path}" if (isUnix()) { sh """#!/bin/bash - "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ } else { bat """ - "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ } } From e1e5c40a5470b9be9ada0959699c9718f77ffeb7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:55:21 +0000 Subject: [PATCH 075/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index a46cd43..8829198 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -43,3 +43,4 @@ def generatelist(checksummable) { } } +return this; From 419d491da023651aaa654db6c9e9f0e9fec1a5db Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 23:09:52 +0000 Subject: [PATCH 076/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 8829198..ef80fdd 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,9 +4,14 @@ env.CHECKSUMARTIFACTS = "checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - def sha256sumfound = true - if (!fileExists("${SHA256SUM}")) { - sha256sumfound = false + def sha256sumfound = false + if (env.SHA256SUM) { + if (fileExists("${SHA256SUM}")) { + sha256sumfound = true + } + } + if (!sha256sumfound) + { if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { @@ -15,9 +20,9 @@ def generatelist(checksummable) { sha256sumfound = true } } - if (!sha256sumfound) { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" - } + } + if (!sha256sumfound) { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ From 29d30dcc7817b2961e4ed97abd67507d73ea97f0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 23:20:01 +0000 Subject: [PATCH 077/589] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 1 - publishing/utils.groovy | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ef80fdd..ccf74db 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -33,7 +33,6 @@ def generatelist(checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { - echo "Checksumming ${file.path}" if (isUnix()) { sh """#!/bin/bash "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 2fdbb73..3fcb1f3 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -20,4 +20,6 @@ def generatelist(publishable) { def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) -} \ No newline at end of file +} + +return this; From 86c18871e6641998816aecbf3cd3461059f970f7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 09:48:30 +0000 Subject: [PATCH 078/589] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 3 ++- publishing/utils.groovy | 3 ++- versionnumbering/metaversion.groovy | 10 ---------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ccf74db..88496b8 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,5 +1,6 @@ -env.CHECKSUMARTIFACTS = "checksums/**" +def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 3fcb1f3..42ebced 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,5 +1,6 @@ -env.PUBLISHABLEARTIFACTS = "publishable/**" +def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index ff20a86..44bfbde 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -2,16 +2,6 @@ def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") -def currentContents(file, newline = "\n", defaultContents = "") { - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - def prefixline(rootpath, filepath) { def fileContents = readFile("${filepath}").trim() def fileContentsNormalized = fileContents.normalize() From 64023d515fc0460ba32cfa5cbb1fead0efa3015a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:24:08 +0000 Subject: [PATCH 079/589] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 meta/folder.groovy diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..d7c91ef --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,10 @@ + +METAFOLDERNAME = "meta" + +def reset() { + dir("${WORKSPACE}/${METAFOLDERNAME}") { + deleteDir() + } +} + +return this; \ No newline at end of file From d835cd4a1b02c781eaeee3751c97fc66988f85c9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:33:10 +0000 Subject: [PATCH 080/589] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index d7c91ef..2f4aaa3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,6 +2,7 @@ METAFOLDERNAME = "meta" def reset() { + echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } From 20697c05f1b45b686302ef82f2139d99dbe51d92 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:43:54 +0000 Subject: [PATCH 081/589] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 88496b8..02a11e0 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -25,7 +25,7 @@ def generatelist(checksummable) { if (!sha256sumfound) { error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } - def CHECKSUMFOLDER = "${WORKSPACE}/checksums" + def CHECKSUMFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/checksums" sh """ rm -rf "${CHECKSUMFOLDER}" mkdir -p "${CHECKSUMFOLDER}" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 42ebced..df7697c 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -4,7 +4,7 @@ env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { - def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" + def PUBLISHABLEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/publishable" sh """ rm -rf "${PUBLISHABLEFOLDER}" mkdir -p "${PUBLISHABLEFOLDER}" From 430fb3349644468bfc03a755894f57556171f627 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:46:49 +0000 Subject: [PATCH 082/589] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 02a11e0..5ec5ba9 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,5 +1,5 @@ -def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // SHA256SUM environment variable must be set to sha256sum utility diff --git a/publishing/utils.groovy b/publishing/utils.groovy index df7697c..72c6566 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,5 +1,5 @@ -def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties From a4d6817925d8094164e4c9f7eb53d5568c68a571 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:57:47 +0000 Subject: [PATCH 083/589] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2f4aaa3..ac82af4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,7 +1,7 @@ METAFOLDERNAME = "meta" -def reset() { +def metareset() { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() From 4d61150d49a920421d46aef3d661630eae2ddb47 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:09:47 +0000 Subject: [PATCH 084/589] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fd15492..89e2444 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -8,7 +8,7 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + stash allowEmpty: true, includes: 'autobuild/**', name: 'Clean workspace stash', useDefaultExcludes: false cleanWs() unstash name: 'Clean workspace stash' } @@ -17,21 +17,6 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { } } -// Start up boilderplate -def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autobuildroot}/Common_scripts") { - echo "Starting...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - - conditionalclean() - - lineendings = load("${commonscriptsroot}/platform/lineendings.groovy") - - def ch = load("${commonscriptsroot}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration() -} - def commonscripts() { return commonscriptsroot } @@ -40,6 +25,24 @@ def getautobuildpath() { return autobuildroot } +// Start up boilderplate +def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + def metafolder = load("${commonscripts()}/meta/folder.groovy") + metafolder.metareset() + + def lineendings = load("${commonscripts()}/platform/lineendings.groovy") + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration() +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } From ec457ffa08b034b2f6baa2db9a2661a57468574e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:11:49 +0000 Subject: [PATCH 085/589] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 89e2444..ef5ea7c 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot { +def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath From de74aae7682505e2c6f1ecbcbb8da7b4c6138cd6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:25:55 +0000 Subject: [PATCH 086/589] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 2 -- versionnumbering/generate.groovy | 12 ++++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ef5ea7c..bff3738 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -37,8 +37,6 @@ def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset() - def lineendings = load("${commonscripts()}/platform/lineendings.groovy") - def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cb5e944..935f026 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,9 +1,11 @@ -env.VERSIONARTIFACTS = "version/**" -VERSIONFOLDER = "${WORKSPACE}/version" +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") + +env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" +VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { - cs = autobuilder.commonscripts() + def cs = autobuilder.commonscripts() if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" @@ -17,7 +19,9 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - configurationhelper.saveConfigurationInVersionInfo(); + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.saveConfigurationInVersionInfo(); } return this; From a59b2c5a58fd6013aa50facb7551aa36cf88aab8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:45:45 +0000 Subject: [PATCH 087/589] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 3 +++ differences/do.groovy | 3 +++ 2 files changed, 6 insertions(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index bff3738..090c0f4 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -45,4 +45,7 @@ def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } +// Everyone wants to know this +autobuilder = this + return this; diff --git a/differences/do.groovy b/differences/do.groovy index b3039a9..7a550a9 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,4 +1,7 @@ +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" + def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() From 11d9fc7ee7386a495d88ac4f397967ed71dacb71 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:48:52 +0000 Subject: [PATCH 088/589] Create a meta folder seperate from other artifacts --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 935f026..f0f93be 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -20,7 +20,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - def ch = load("${commonscripts()}/configuration/configuration.groovy") + def ch = load("${cs}/configuration/configuration.groovy") ch.saveConfigurationInVersionInfo(); } From 387b799b47c8af15f9c8308b37fe5dfe4b79a797 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:16:23 +0000 Subject: [PATCH 089/589] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 41 +++---------------------- differences/do.groovy | 6 ++-- differences/show_release_differences.py | 20 ++++++------ meta/folder.groovy | 1 + 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 5ec5ba9..add3a50 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -2,50 +2,19 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" -// SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - def sha256sumfound = false - if (env.SHA256SUM) { - if (fileExists("${SHA256SUM}")) { - sha256sumfound = true - } - } - if (!sha256sumfound) - { - if (isUnix()) { - def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() - if (sha256sumpath) { - echo "Found sha256sum executable here: ${sha256sumpath}" - env.SHA256SUM = sha256sumpath - sha256sumfound = true - } - } - } - if (!sha256sumfound) { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" - } - def CHECKSUMFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/checksums" - sh """ - rm -rf "${CHECKSUMFOLDER}" - mkdir -p "${CHECKSUMFOLDER}" - """ + sums = "" for (filepath in checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { - if (isUnix()) { - sh """#!/bin/bash - "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" - """ - } - else { - bat """ - "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" - """ - } + def sum = sha1(file: file.path) + echo "Sum from file ${file.path} is ${sum}" + sums += "${sum}${nl}" } } + writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) } return this; diff --git a/differences/do.groovy b/differences/do.groovy index 7a550a9..8866400 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,17 +2,17 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEARTIFACTS}") { cs = autobuilder.commonscripts() if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 2d3ee0d..0813356 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,9 +29,9 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(versionfolder): - print("Creating folder (if not already there): {0}".format(versionfolder)) - os.makedirs(versionfolder, exist_ok = True) +def makeFolder(difffolder): + print("Creating folder (if not already there): {0}".format(difffolder)) + os.makedirs(difffolder, exist_ok = True) def printDivider(outfh, times = 1, length = 100): for _ in range(times): @@ -172,7 +172,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +183,14 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - versionfolder = args.versionfolder + difffolder = args.difffolder print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) - if versionfolder: - makeFolder(versionfolder) - outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + if difffolder: + makeFolder(difffolder) + outfh = open(os.path.join(difffolder, "differences.txt"), 'w') else: outfh = sys.stdout @@ -209,7 +209,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if versionfolder: + if difffolder: outfh.close() return 0 except Exception as e: diff --git a/meta/folder.groovy b/meta/folder.groovy index ac82af4..6b0b21c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ METAFOLDERNAME = "meta" +env.METAARTIFACTS = "${METAFOLDERNAME}/**" def metareset() { echo "Resetting meta folder" From 552370f9c1126e45b772762b274462ad26bd412b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:23:39 +0000 Subject: [PATCH 090/589] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index add3a50..ec3b10a 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,14 +4,14 @@ env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { + load("${autobuilder.commonscripts()}/platform/lineendings.groovy") sums = "" for (filepath in checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { def sum = sha1(file: file.path) - echo "Sum from file ${file.path} is ${sum}" - sums += "${sum}${nl}" + sums += "${sum} ${file.path}${nl}" // Format suitable for sha1sum utility } } writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) From 92e52dac5af83f9d04cec0f8abbe673cea445bf3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:27:40 +0000 Subject: [PATCH 091/589] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 1 + versionnumbering/generate.groovy | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ec3b10a..748472f 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,6 +1,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" +CHECKSUMFOLDER = "${metafolder.METAFOLDERNAME}/checksums" // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f0f93be..cdcf1c2 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,6 +1,5 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") - env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" From 0ad7692aed22ea93a9bd3e9b956c96e23cffbd27 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:36:24 +0000 Subject: [PATCH 092/589] Create a meta folder seperate from other artifacts --- differences/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 8866400..9e572af 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,10 +1,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" +DIFFERENCEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/differences" -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEARTIFACTS}") { +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEFOLDER}") { cs = autobuilder.commonscripts() - if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" From c04c82be175e05d9cbe7bc8f37f3d22538097a6b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 21:23:41 +0000 Subject: [PATCH 093/589] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 5 ++++- product/info.groovy | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 product/info.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 090c0f4..fc298ba 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -37,6 +37,9 @@ def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset() + def productinfo = load("${commonscripts()}/product/info.groovy") + productinfo.setname(productname, jirakey) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() } diff --git a/product/info.groovy b/product/info.groovy new file mode 100644 index 0000000..c41fc80 --- /dev/null +++ b/product/info.groovy @@ -0,0 +1,16 @@ + +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" +env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" + +def setname(productname = "", jirakey = "") { + def productdata = [ + 'name' : productname.trim(), + 'jirakey' : jirakey.trim() + ] + def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) + def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" + writeJSON(file: PRODUCTFILE, json: outJson, pretty: 4) +} + +return this; \ No newline at end of file From 0b491063dcd3de67fbe2c5e3543314ceb0d3f818 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 10:52:04 +0000 Subject: [PATCH 094/589] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 4 ++-- product/info.groovy | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fc298ba..ae3af20 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -38,7 +38,7 @@ def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commons metafolder.metareset() def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(productname, jirakey) + productinfo.setname(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/product/info.groovy b/product/info.groovy index c41fc80..05ed92a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -3,9 +3,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -def setname(productname = "", jirakey = "") { +def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { def productdata = [ 'name' : productname.trim(), + 'subcomponent' : subcomponentname.trim() 'jirakey' : jirakey.trim() ] def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) From f8bfaca3f08583a03dd077920637ef6b912b1a44 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 10:53:23 +0000 Subject: [PATCH 095/589] Create a meta folder seperate from other artifacts --- product/info.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 05ed92a..a908e7e 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -6,8 +6,8 @@ env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { def productdata = [ 'name' : productname.trim(), - 'subcomponent' : subcomponentname.trim() - 'jirakey' : jirakey.trim() + 'subcomponent' : subcomponentname.trim(), + 'jirakey' : jirakey.trim(), ] def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" From d1b409b2d49953164cd5f30524832285c8dfd521 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:32:14 +0000 Subject: [PATCH 096/589] Create a meta folder seperate from other artifacts --- product/info.groovy | 40 ++++++++++++--- versionnumbering/generate.groovy | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index a908e7e..71cf6fa 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,17 +1,43 @@ +load("${autobuilder.commonscripts()}/json/interface.groovy") + metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" +class ProductInfo extends JsonSerialised { + private String productname + private String subcomponentname + private String jirakey + + ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + productname = productName + subcomponentname = subcomponentName + jirakey = jiraKey + } + + def serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +theProductInfo = new ProductInfo() + def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - def productdata = [ - 'name' : productname.trim(), - 'subcomponent' : subcomponentname.trim(), - 'jirakey' : jirakey.trim(), - ] - def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) + theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) + def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - writeJSON(file: PRODUCTFILE, json: outJson, pretty: 4) + theProductInfo.saveJsonSerialised(PRODUCTFILE) } return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cdcf1c2..ad6214d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -3,6 +3,89 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +class GitInfo { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo +} + +class JenkinsBuildInfo { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate +} + +class BuildStampInfo { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep +} + +class ReleaseInfo { + private String releasejobname +} + +class VersionInfo { + private ComponentInfo componentinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + + private VersionInfo[] associatedversions +} + + def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { + + + version.generate("${SOURCE_REVISION}", "${SOURCE_PATH}", "${BASEBUILDNUMBER}", "${RELEASE_JOB_NAME}") + + VersionInfo() { + } + def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + + with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: + f.write(str(datetime.datetime.now())) + + with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: + f.write(releasejobname) + +} + def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From ae3ad5f31d3b6a57c261b18345069a9f0fa17520 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:34:36 +0000 Subject: [PATCH 097/589] Create a meta folder seperate from other artifacts --- json/interface.groovy | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 json/interface.groovy diff --git a/json/interface.groovy b/json/interface.groovy new file mode 100644 index 0000000..8962578 --- /dev/null +++ b/json/interface.groovy @@ -0,0 +1,7 @@ + +abstract class JsonSerialised { + def serialisationdata() + def saveJsonSerialised(String outputfile, int prettylevel = 4) +} + +return this; From bba686364164d7f8e781837ee06aa068e842d175 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:39:03 +0000 Subject: [PATCH 098/589] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 71cf6fa..979d0b5 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,5 +1,5 @@ -load("${autobuilder.commonscripts()}/json/interface.groovy") +def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" From f8f739e6fdd7b6efad937cc7ea60121541c02fdc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:22:46 +0000 Subject: [PATCH 099/589] Create a meta folder seperate from other artifacts --- json/interface.groovy | 4 ++-- product/info.groovy | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index 8962578..d238707 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,7 +1,7 @@ abstract class JsonSerialised { - def serialisationdata() - def saveJsonSerialised(String outputfile, int prettylevel = 4) + def serialisationdata(); + def saveJsonSerialised(String outputfile, int prettylevel = 4); } return this; diff --git a/product/info.groovy b/product/info.groovy index 979d0b5..9b2551c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -11,6 +11,7 @@ class ProductInfo extends JsonSerialised { private String jirakey ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + super() productname = productName subcomponentname = subcomponentName jirakey = jiraKey From babca972e743871e9e1178aafc6d4380a2520691 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:25:14 +0000 Subject: [PATCH 100/589] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 9b2551c..783b84c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo extends JsonSerialised { +class ProductInfo extends ji.JsonSerialised { private String productname private String subcomponentname private String jirakey From 5d05f9baeab64949a471f92cfc315dc72ee7028d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:26:48 +0000 Subject: [PATCH 101/589] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 783b84c..8b8d3e2 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo extends ji.JsonSerialised { +class ProductInfo { private String productname private String subcomponentname private String jirakey From 210e17c2e88acc28edc3ec3c2fa63da4c8ce77d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:29:20 +0000 Subject: [PATCH 102/589] Create a meta folder seperate from other artifacts --- json/interface.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index d238707..4cd2628 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,7 +1,7 @@ abstract class JsonSerialised { - def serialisationdata(); - def saveJsonSerialised(String outputfile, int prettylevel = 4); + abstract void serialisationdata(); + abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } return this; From 6ed97583368d97dd9badd659c8429a86b94de1e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:37:04 +0000 Subject: [PATCH 103/589] Create a meta folder seperate from other artifacts --- product/info.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 8b8d3e2..536d336 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -26,9 +26,9 @@ class ProductInfo { return data } - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + def saveJsonSerialised(script, String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) } } @@ -38,7 +38,7 @@ def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, ji theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(PRODUCTFILE) + theProductInfo.saveJsonSerialised(this, PRODUCTFILE) } return this; \ No newline at end of file From bf13a8f5a3b35f461e2fca45c5232055154f1aa5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 16:03:24 +0000 Subject: [PATCH 104/589] Create a meta folder seperate from other artifacts --- json/interface.groovy | 26 +++++++++++ versionnumbering/generate.groovy | 75 +------------------------------- 2 files changed, 28 insertions(+), 73 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index 4cd2628..fae1fe3 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -4,4 +4,30 @@ abstract class JsonSerialised { abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } +class GitInfo { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo +} + +class JenkinsBuildInfo { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate +} + +class BuildStampInfo { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep +} + +class ReleaseInfo { + private String releasejobname +} + return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ad6214d..3d498e9 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,34 +1,10 @@ +def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") + metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -class GitInfo { - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo -} - -class JenkinsBuildInfo { - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate -} - -class BuildStampInfo { - private String buildstampprefix - private int revision - private String buildstamp - private String buildstampcommasep -} - -class ReleaseInfo { - private String releasejobname -} - class VersionInfo { private ComponentInfo componentinfo private GitInfo gitinfo @@ -37,53 +13,6 @@ class VersionInfo { private ReleaseInfo releaseinfo private VersionInfo[] associatedversions -} - - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { - - - version.generate("${SOURCE_REVISION}", "${SOURCE_PATH}", "${BASEBUILDNUMBER}", "${RELEASE_JOB_NAME}") - - VersionInfo() { - } - def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - - with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: - f.write(releasejobname) - } def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { From efdf4ae8b854e64b0109205347cb03779a1c3ba1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 16:17:20 +0000 Subject: [PATCH 105/589] Create a meta folder seperate from other artifacts --- json/interface.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index fae1fe3..c9cd9f9 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -4,7 +4,7 @@ abstract class JsonSerialised { abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } -class GitInfo { +public class GitInfo { private String sourcepath private String sourcerevision private String gitrevision @@ -12,21 +12,21 @@ class GitInfo { private String remoteinfo } -class JenkinsBuildInfo { +public class JenkinsBuildInfo { private String buildjobname private int buildnumber private String buildmachine private String builddate } -class BuildStampInfo { +public class BuildStampInfo { private String buildstampprefix private int revision private String buildstamp private String buildstampcommasep } -class ReleaseInfo { +public class ReleaseInfo { private String releasejobname } From 7d0b8095a30fde7023129efd378b96a78957e83a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:30:30 +0000 Subject: [PATCH 106/589] Pass script object to custom objects --- autobuilder.groovy | 4 ++-- product/info.groovy | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ae3af20..ffaaf9f 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -38,7 +38,7 @@ def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jira metafolder.metareset() def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(productname, subcomponentname, jirakey) + productinfo.setname(script, productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/product/info.groovy b/product/info.groovy index 536d336..1e0af6d 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,12 +5,14 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo { +class ProductInfo implements Serializable { + private script private String productname private String subcomponentname private String jirakey - ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { + this.script = script super() productname = productName subcomponentname = subcomponentName @@ -26,7 +28,7 @@ class ProductInfo { return data } - def saveJsonSerialised(script, String outputfile, int prettylevel = 4) { + def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) } @@ -34,11 +36,11 @@ class ProductInfo { theProductInfo = new ProductInfo() -def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) +def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { + theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(this, PRODUCTFILE) + theProductInfo.saveJsonSerialised(PRODUCTFILE) } return this; \ No newline at end of file From 518b60a5e8f83136d3168b84c2ec993c467fd1fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:32:24 +0000 Subject: [PATCH 107/589] Pass script object to custom objects --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 1e0af6d..19f5df0 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -12,8 +12,8 @@ class ProductInfo implements Serializable { private String jirakey ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { - this.script = script super() + this.script = script productname = productName subcomponentname = subcomponentName jirakey = jiraKey From ce0643254bbbd99ef8bc4dbdca6eb34136a585c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:34:40 +0000 Subject: [PATCH 108/589] Pass script object to custom objects --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 19f5df0..78e281a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -34,7 +34,7 @@ class ProductInfo implements Serializable { } } -theProductInfo = new ProductInfo() +theProductInfo = null def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) From 05284a7607fbf8f3ce9b3b0c560047d576af942d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:52:07 +0000 Subject: [PATCH 109/589] Pass script object to custom objects --- versionnumbering/generate.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3d498e9..3d35e23 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -5,8 +5,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +def pi = load("${autobuilder.commonscripts()}/product/info.groovy") + class VersionInfo { - private ComponentInfo componentinfo + private ProductInfo productinfo private GitInfo gitinfo private JenkinsBuildInfo jenkinsbuildinfo private BuildStampInfo buildstampinfo From a1a36f47b7d91313a0d30a61aee7f98af34a1be8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 18:25:17 +0000 Subject: [PATCH 110/589] Pass script object to custom objects --- json/interface.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json/interface.groovy b/json/interface.groovy index c9cd9f9..58f9eb7 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,5 +1,5 @@ -abstract class JsonSerialised { +abstract class GISJsonSerialised { abstract void serialisationdata(); abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } From f4753020d9c0e901b0343ac4d7ff195ffc4a40c2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:18:27 +0000 Subject: [PATCH 111/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- autobuilder.groovy | 6 +- json/interface.groovy | 33 ------ meta/folder.groovy | 180 ++++++++++++++++++++++++++++++- product/info.groovy | 46 -------- versionnumbering/generate.groovy | 13 --- 5 files changed, 181 insertions(+), 97 deletions(-) delete mode 100644 json/interface.groovy delete mode 100644 product/info.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index ffaaf9f..dfc470b 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -35,10 +35,8 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() def metafolder = load("${commonscripts()}/meta/folder.groovy") - metafolder.metareset() - - def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(script, productname, subcomponentname, jirakey) + metafolder.metareset(script) + metafolder.setproductinfo(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/json/interface.groovy b/json/interface.groovy deleted file mode 100644 index 58f9eb7..0000000 --- a/json/interface.groovy +++ /dev/null @@ -1,33 +0,0 @@ - -abstract class GISJsonSerialised { - abstract void serialisationdata(); - abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); -} - -public class GitInfo { - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo -} - -public class JenkinsBuildInfo { - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate -} - -public class BuildStampInfo { - private String buildstampprefix - private int revision - private String buildstamp - private String buildstampcommasep -} - -public class ReleaseInfo { - private String releasejobname -} - -return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 6b0b21c..a8bc619 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,11 +2,189 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" -def metareset() { +class GISJsonSerialised implements Serializable { + protected script + + GISJsonSerialied(script) { + this.script = script + } + + abstract void serialisationdata(); + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + private String productname + private String subcomponentname + private String jirakey + + def set(script, productname, subcomponentname, jirakey) { + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jiraKey + } + + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + set(productname, subcomponentname, jirakey) + } + + def serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + super(script) + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + def serialisationdata() { + def data = [ + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + + JenkinsBuildInfo(script, buildjobname = "", buildnumber = 0, buildmachine = "", builddate = "") { + super(script) + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + def serialisationdata() { + def data = [ + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep + + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + super(script) + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + + def serialisationdata() { + def data = [ + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + 'buildstampcommasep' : buildstampcommasep, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + private String releasejobname + + BuildStampInfo(script, releasejobname = "") { + super(script) + this.releasejobname = releasejobname + } + + def serialisationdata() { + def data = [ + 'releasejobname' : releasejobname, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + + private MetaInfo[] associatedmeta + + MetaInfo(script) { + super(script) + productinfo(script) + gitinfo(script) + jenkinsbuildinfo(script) + buildstampinfo(script) + releaseinfo(script) + } + + def serialisationdata() { + def data = [ + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + ] + return datasetProductInfo + } + + def setproductinfo(productname, subcomponentname, jirakey) { + productinfo.set(productname, subcomponentname, jirakey) + } + + +MetaInfo theMetaData = new MetaInfo(this) + +def metareset(script) { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } + theMetaData = new MetaInfo(this) } +def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { + theMetaData.setproductinfo(productname, subcomponentname, jirakey) +} + + + return this; \ No newline at end of file diff --git a/product/info.groovy b/product/info.groovy deleted file mode 100644 index 78e281a..0000000 --- a/product/info.groovy +++ /dev/null @@ -1,46 +0,0 @@ - -def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") - -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" -env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" - -class ProductInfo implements Serializable { - private script - private String productname - private String subcomponentname - private String jirakey - - ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { - super() - this.script = script - productname = productName - subcomponentname = subcomponentName - jirakey = jiraKey - } - - def serialisationdata() { - def data = [ - 'productname' : productname, - 'subcomponentname' : subcomponentname, - 'jirakey' : jirakey, - ] - return data - } - - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) - } -} - -theProductInfo = null - -def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) - - def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(PRODUCTFILE) -} - -return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3d35e23..48f8bb6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,22 +1,9 @@ -def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") - metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" def pi = load("${autobuilder.commonscripts()}/product/info.groovy") -class VersionInfo { - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo - - private VersionInfo[] associatedversions -} - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From df5d11caa3f562bc5062d58aa50cf78b72a7784e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:37:27 +0000 Subject: [PATCH 112/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- versionnumbering/generate.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 48f8bb6..b576201 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -2,8 +2,6 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -def pi = load("${autobuilder.commonscripts()}/product/info.groovy") - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From 6323d8f5ebd5d1775da703bd93aadb78a42e31ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:45:39 +0000 Subject: [PATCH 113/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8bc619..0e800a4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -3,7 +3,7 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" class GISJsonSerialised implements Serializable { - protected script + def script GISJsonSerialied(script) { this.script = script From 397727d4f1bec0be03c5d7feaac10cea5f5a116a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:47:17 +0000 Subject: [PATCH 114/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0e800a4..ca4680f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -5,7 +5,7 @@ env.METAARTIFACTS = "${METAFOLDERNAME}/**" class GISJsonSerialised implements Serializable { def script - GISJsonSerialied(script) { + GISJsonSerialised(script) { this.script = script } From 746838240f0296d955740836debf602c820478a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:49:13 +0000 Subject: [PATCH 115/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ca4680f..65d4708 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -124,7 +124,7 @@ class BuildStampInfo extends GISJsonSerialised { class ReleaseInfo extends GISJsonSerialised { private String releasejobname - BuildStampInfo(script, releasejobname = "") { + ReleaseInfo(script, releasejobname = "") { super(script) this.releasejobname = releasejobname } From bba626b4175213c090a813b9560b12bce63630b6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:51:29 +0000 Subject: [PATCH 116/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 65d4708..46ae4f3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -169,7 +169,7 @@ class MetaInfo extends GISJsonSerialised { def setproductinfo(productname, subcomponentname, jirakey) { productinfo.set(productname, subcomponentname, jirakey) } - +} MetaInfo theMetaData = new MetaInfo(this) @@ -185,6 +185,4 @@ def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_N theMetaData.setproductinfo(productname, subcomponentname, jirakey) } - - return this; \ No newline at end of file From 326a38f0879ebbae0cd1e7743cabd659c8e5e09a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:03:51 +0000 Subject: [PATCH 117/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 46ae4f3..f13cdaa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,14 +2,14 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" -class GISJsonSerialised implements Serializable { +abstract class GISJsonSerialised implements Serializable { def script GISJsonSerialised(script) { this.script = script } - abstract void serialisationdata(); + abstract Map serialisationdata(); def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -33,7 +33,7 @@ class ProductInfo extends GISJsonSerialised { set(productname, subcomponentname, jirakey) } - def serialisationdata() { + Map serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, @@ -59,7 +59,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - def serialisationdata() { + Map serialisationdata() { def data = [ 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, @@ -85,7 +85,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - def serialisationdata() { + Map serialisationdata() { def data = [ 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, @@ -110,7 +110,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - def serialisationdata() { + Map serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, 'revision' : revision, @@ -129,7 +129,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - def serialisationdata() { + Map serialisationdata() { def data = [ 'releasejobname' : releasejobname, ] @@ -153,15 +153,25 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo(script) buildstampinfo(script) releaseinfo(script) + associatedmeta = [] } - def serialisationdata() { + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(serialisationdata(metaitem)) + } + return associated + } + + Map serialisationdata() { def data = [ 'productinfo' : productinfo.serialisationdata(), 'gitinfo' : gitinfo.serialisationdata(), 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), + 'associatedmeta' : associatedmetaserialised(), ] return datasetProductInfo } From c48b95b3f692ceb39d81c2409ba39eedf19dff64 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:38:55 +0000 Subject: [PATCH 118/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f13cdaa..afcd6f7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -148,11 +148,11 @@ class MetaInfo extends GISJsonSerialised { MetaInfo(script) { super(script) - productinfo(script) - gitinfo(script) - jenkinsbuildinfo(script) - buildstampinfo(script) - releaseinfo(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfoo = new JenkinsBuildInfo(script) + buildstampinfoo = new BuildStampInfo(script) + releaseinfoo = new ReleaseInfo(script) associatedmeta = [] } From b4dd72684c56a7f4d27d652f85e922c28a9b63ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:41:46 +0000 Subject: [PATCH 119/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index afcd6f7..ad3636d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -22,7 +22,7 @@ class ProductInfo extends GISJsonSerialised { private String subcomponentname private String jirakey - def set(script, productname, subcomponentname, jirakey) { + def set(productname, subcomponentname, jirakey) { this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jiraKey From 98311b6e4f2640da54ff0c021867261564e05a94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 19:38:29 +0000 Subject: [PATCH 120/589] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- autobuilder.groovy | 2 +- meta/folder.groovy | 62 ++++++++++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index dfc470b..fab812a 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -36,7 +36,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset(script) - metafolder.setproductinfo(productname, subcomponentname, jirakey) + metafolder.theMetaData.productinfo.set(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/meta/folder.groovy b/meta/folder.groovy index ad3636d..d4caa5d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -9,7 +9,7 @@ abstract class GISJsonSerialised implements Serializable { this.script = script } - abstract Map serialisationdata(); + abstract LinkedHashMap serialisationdata(); def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -22,18 +22,20 @@ class ProductInfo extends GISJsonSerialised { private String subcomponentname private String jirakey + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + def set(productname, subcomponentname, jirakey) { this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jiraKey + this.jirakey = jirakey } - ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { - super(script) - set(productname, subcomponentname, jirakey) - } - - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, @@ -59,7 +61,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, @@ -85,7 +87,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, @@ -110,7 +112,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, 'revision' : revision, @@ -129,7 +131,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'releasejobname' : releasejobname, ] @@ -138,34 +140,40 @@ class ReleaseInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo + private String format = '0.1' + ProductInfo productinfo + GitInfo gitinfo + JenkinsBuildInfo jenkinsbuildinfo + BuildStampInfo buildstampinfo + ReleaseInfo releaseinfo - private MetaInfo[] associatedmeta + private ArrayList associatedmeta MetaInfo(script) { super(script) productinfo = new ProductInfo(script) gitinfo = new GitInfo(script) - jenkinsbuildinfoo = new JenkinsBuildInfo(script) - buildstampinfoo = new BuildStampInfo(script) - releaseinfoo = new ReleaseInfo(script) - associatedmeta = [] + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + associatedmeta = new ArrayList() } List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { - associated.add(serialisationdata(metaitem)) + associated.add(metaitem.serialisationdata()) } return associated } - Map serialisationdata() { + void addAssociatedMetaInfo(mi) { + associatedmeta.add(mi) + } + + LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'productinfo' : productinfo.serialisationdata(), 'gitinfo' : gitinfo.serialisationdata(), 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), @@ -173,7 +181,7 @@ class MetaInfo extends GISJsonSerialised { 'releaseinfo' : releaseinfo.serialisationdata(), 'associatedmeta' : associatedmetaserialised(), ] - return datasetProductInfo + return data } def setproductinfo(productname, subcomponentname, jirakey) { @@ -191,8 +199,4 @@ def metareset(script) { theMetaData = new MetaInfo(this) } -def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theMetaData.setproductinfo(productname, subcomponentname, jirakey) -} - return this; \ No newline at end of file From 59d37c79a58e01bef069a8e8f395677395aa8105 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 20:25:33 +0000 Subject: [PATCH 121/589] Start using meta info for storing configuration --- autobuilder.groovy | 13 ++++---- configuration/configuration.groovy | 33 ++++---------------- meta/folder.groovy | 48 ++++++++++++++++++++++++++---- versionnumbering/generate.groovy | 3 -- 4 files changed, 57 insertions(+), 40 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fab812a..a82e891 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolder = load("${commonscripts()}/meta/folder.groovy") + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -26,7 +28,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -34,12 +36,13 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - metafolder.metareset(script) - metafolder.theMetaData.productinfo.set(productname, subcomponentname, jirakey) + def meta = metafolder.metareset(script) + meta.productinfo.set(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration() + ch.registerPipelineScriptConfiguration(meta) + + return meta } def configurationhelper() { diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 1f0c095..30b7086 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,37 +1,16 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -configurationCollection = [:] - -def registerConfigurationInfo(String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - def ConfigurationInfo = [configurationURL, configurationVersion, configurationSHA] - configurationCollection.put(configurationName, ConfigurationInfo) +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) } -def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) +def registerPipelineScriptConfiguration(meta) { + registerConfigurationInfo(meta, 'pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } -def saveConfigurationInVersionInfo() { - dir("${WORKSPACE}/version") { - echo "Writing configuration info to ${WORKSPACE}/version/configuration.txt" - writeFile file: 'configuration.txt', text: getFormattedConfiguration() - } -} - -String getFormattedConfiguration() { - String formattedInfo = "" - def keys = configurationCollection.keySet().sort() - for (key in keys) { - (configurationURL, configurationVersion, configurationSHA) = configurationCollection.get(key) - formattedLine = String.format("%s:%s,%s,%s%n", key, configurationURL, configurationVersion, configurationSHA) - formattedInfo += formattedLine - } - return formattedInfo -} - -def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(tool, gitURL, revision, gh.getGitInfo(targetDir)) + registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this diff --git a/meta/folder.groovy b/meta/folder.groovy index d4caa5d..2963e58 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -139,6 +139,31 @@ class ReleaseInfo extends GISJsonSerialised { } } +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String format = '0.1' ProductInfo productinfo @@ -146,7 +171,7 @@ class MetaInfo extends GISJsonSerialised { JenkinsBuildInfo jenkinsbuildinfo BuildStampInfo buildstampinfo ReleaseInfo releaseinfo - + private ArrayList configurationinfo private ArrayList associatedmeta MetaInfo(script) { @@ -156,9 +181,23 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo = new JenkinsBuildInfo(script) buildstampinfo = new BuildStampInfo(script) releaseinfo = new ReleaseInfo(script) + configurationinfo = new ArrayList() associatedmeta = new ArrayList() } + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return associated + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + configurationinfo.add(ci) + } + List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { @@ -179,6 +218,7 @@ class MetaInfo extends GISJsonSerialised { 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), 'associatedmeta' : associatedmetaserialised(), ] return data @@ -189,14 +229,12 @@ class MetaInfo extends GISJsonSerialised { } } -MetaInfo theMetaData = new MetaInfo(this) - -def metareset(script) { +MetaInfo metareset(script) { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } - theMetaData = new MetaInfo(this) + return new MetaInfo(script) } return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index b576201..b99467e 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -17,9 +17,6 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - - def ch = load("${cs}/configuration/configuration.groovy") - ch.saveConfigurationInVersionInfo(); } return this; From 35ae72bbdbbf7c34bdf47921be41b6b7522c7dd2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 20:29:56 +0000 Subject: [PATCH 122/589] Start using meta info for storing configuration --- autobuilder.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index a82e891..e7943c6 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,8 +4,6 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" -metafolder = load("${commonscripts()}/meta/folder.groovy") - def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -28,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -36,6 +34,7 @@ MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONE conditionalclean() + def metafolder = load("${commonscripts()}/meta/folder.groovy") def meta = metafolder.metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) From d9b91dee691a29ae78466b8cbc2fd528d9d9a745 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 22:52:20 +0000 Subject: [PATCH 123/589] Move functionality to meta --- autobuilder.groovy | 5 ++++- meta/folder.groovy | 13 ++++++++++++- versionnumbering/generate.groovy | 32 +++++++++++++++++++++++++++++--- versionnumbering/versioninfo.py | 25 +++---------------------- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index e7943c6..3a2a408 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -37,7 +37,10 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def metafolder = load("${commonscripts()}/meta/folder.groovy") def meta = metafolder.metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) - + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2963e58..6ae3d3d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -79,7 +79,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { private String buildmachine private String builddate - JenkinsBuildInfo(script, buildjobname = "", buildnumber = 0, buildmachine = "", builddate = "") { + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { super(script) this.buildjobname = buildjobname this.buildnumber = buildnumber @@ -87,6 +87,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } + def set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, @@ -131,6 +138,10 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } + def set(releasejobname) { + this.releasejobname = releasejobname + } + LinkedHashMap serialisationdata() { def data = [ 'releasejobname' : releasejobname, diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index b99467e..0019b24 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -2,16 +2,16 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { +def generate(meta, sourcerevision, sourcepath, basebuildnumber = "1000", versionfolder = "${VERSIONFOLDER}") { def cs = autobuilder.commonscripts() if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" """ } env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() @@ -20,3 +20,29 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } return this; + + +/* + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) +*/ diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 370d9fc..0fa097a 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -24,7 +24,6 @@ __date__ = '2017-11-29' __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -slaveMachine = os.getenv("NODE_NAME", default="-") MAXVERSIONNUMBER = 65535 def resetFolder(versionfolder): @@ -138,10 +137,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - +def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: f.write(versionprefix) @@ -157,9 +153,6 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: f.write(remoteurl) - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: f.write(buildstamp) @@ -167,15 +160,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: f.write(buildstampcommasep) - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: f.write(str(datetime.datetime.now())) - with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: - f.write(releasejobname) - def main(argv=None): # IGNORE:C0111 '''Command line options.''' @@ -207,9 +194,6 @@ USAGE parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") - parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") - parser.add_argument("-r", "--releasejobname", dest="releasejobname", action="store", default="-", help="release-to-git build job name [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -219,12 +203,9 @@ USAGE versionfolder = args.versionfolder gitrootfolder = args.gitrootfolder baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber - releasejobname = args.releasejobname print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber, "Release job name:", releasejobname) + print("Base version number:", baseversionnumber) resetFolder(versionfolder) @@ -235,7 +216,7 @@ USAGE buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname) + writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) return 0 except Exception as e: From 84c105946be4abdcb35ad85da8b847e81875a511 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:08:11 +0000 Subject: [PATCH 124/589] Improve meta gathering --- meta/folder.groovy | 15 ++++ versionnumbering/generate.groovy | 134 +++++++++++++++++++++++-------- versionnumbering/versioninfo.py | 53 ------------ 3 files changed, 114 insertions(+), 88 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6ae3d3d..8b638a9 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -61,6 +61,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + def set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, @@ -119,6 +127,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + def set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 0019b24..f9683e6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,48 +1,112 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +ddef getGitRevision(String gitrootpath, int basebuildnumber) { + dir( gitrootpath) { + def int MAXVERSIONNUMBER = 65535 -def generate(meta, sourcerevision, sourcepath, basebuildnumber = "1000", versionfolder = "${VERSIONFOLDER}") { - def cs = autobuilder.commonscripts() - if (isUnix()) { - sh """ - "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" - """ + def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count") + echo "Change count: ${changecount}" + def int adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + echo "Adjusted change count: ${adjustedchangecount}" + + def gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD") + echo "Revision: ${gitrevision}" + + def remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin") + echo "Remote info: ${remoteinfo}" + + def remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin") + echo "Remote URL: ${remoteurl}" } - else { - bat """ - "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" - """ - } - env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() - env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() - env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] } -return this; +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch) { + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + def defaultversionprefix = "1.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } -/* - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) + def mastertypeversionprefix = "1.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) + echo "sourcebranch is now: ${sourcebranch}" + def splits2 = sourcebranch.split('\\.') + echo "Splits2 is: ${splits2}" + if (splits2.size() < 1) { + def thirdpartyversionprefix = "1.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + splits = [] + for (split in splits2) { + echo "Split is: ${split}" + if (split.isInteger()) { + splits.add(split) + } + else if (split == "x") { + return mastertypeversionprefix + } + } + echo "Splits is: ${splits}" - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) + def versionprefix = "" + if (splits.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int splitnumbercount = 0 + for (split in splits) { + echo "Split is ${split}" + if (split.isInteger()) { + versionprefix += split + "." + splitnumbercount += 1 + } + else if (splitnumbercount > 2) { + break + } + } + if (splitnumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + if (splitnumbercount == 1) { + versionprefix += "0.0." + } + else if (splitnumbercount == 2) { + versionprefix += "0." + } + } + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) + def versionprefix = getVersionPrefix(sourcerevision) - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) -*/ + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + + return buildstamp +} + +return this; \ No newline at end of file diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 0fa097a..3d2a80d 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -23,57 +23,6 @@ __version__ = 0.1 __date__ = '2017-11-29' __updated__ = '2017-11-29' -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -MAXVERSIONNUMBER = 65535 - -def resetFolder(versionfolder): - print("Removing folders and contents of: {0}".format(versionfolder)) - try: - shutil.rmtree(versionfolder) - except FileNotFoundError: - pass - - print("Creating folder: {0}".format(versionfolder)) - os.makedirs(versionfolder) - -def getGitRevision(gitrootpath, basenumber): - cwd = os.getcwd() - - path = os.path.expanduser(gitrootpath) - print("Changing current working directory to: {0}".format(path)) - os.chdir(path) - - print("Determining revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - changecount = completed.stdout.decode("utf-8").strip() - print("Change count: {0}".format(changecount)) - adjustedchangecount = int((int(changecount) + int(basenumber)) % (MAXVERSIONNUMBER + 1)) - print("Adjusted change count: {0}".format(adjustedchangecount)) - - print("Determining current revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - revision = completed.stdout.decode("utf-8").strip() - print("Revision: {0}".format(revision)) - - print("Determining remote info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteinfo = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteinfo)) - - print("Determining remote url using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "get-url", "--push", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteurl = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteurl)) - - os.chdir(cwd) - print("Restored current working directory to: {0}".format(cwd)) - - return (adjustedchangecount, revision, remoteinfo, remoteurl) - # Handle: # origin/master # origin/2.0.x @@ -207,8 +156,6 @@ USAGE print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) print("Base version number:", baseversionnumber) - resetFolder(versionfolder) - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) versionprefix = getVersionPrefix(sourcebranch) From 654b4fa87be1ba9de08f3aefcd5793740f2391bd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:08:43 +0000 Subject: [PATCH 125/589] Improve meta gathering --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f9683e6..69bdd4d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,5 +1,5 @@ -ddef getGitRevision(String gitrootpath, int basebuildnumber) { +def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 From 963dabcc6e61dd6a97e76f5243be4786789d1ca9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:13:47 +0000 Subject: [PATCH 126/589] Improve meta gathering --- meta/folder.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8b638a9..690ffaa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -29,7 +29,7 @@ class ProductInfo extends GISJsonSerialised { this.jirakey = jirakey } - def set(productname, subcomponentname, jirakey) { + void set(String productname, String subcomponentname, String jirakey) { this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jirakey @@ -61,7 +61,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - def set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) { + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -95,7 +95,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - def set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine @@ -127,7 +127,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - def set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp @@ -153,7 +153,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - def set(releasejobname) { + void set(String releasejobname) { this.releasejobname = releasejobname } @@ -220,7 +220,7 @@ class MetaInfo extends GISJsonSerialised { } void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { - ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) configurationinfo.add(ci) } @@ -263,4 +263,4 @@ MetaInfo metareset(script) { return new MetaInfo(script) } -return this; \ No newline at end of file +return this; From 0fe07ee41231719f3b8a42b7fd089ac590a09602 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:22:00 +0000 Subject: [PATCH 127/589] Improve meta gathering --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 690ffaa..045d67d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -216,7 +216,7 @@ class MetaInfo extends GISJsonSerialised { for (configurationitem in configurationinfo) { configuration.add(configurationitem.serialisationdata()) } - return associated + return configuration } void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 69bdd4d..3c475bc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,20 +1,24 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count") + def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count").trim() echo "Change count: ${changecount}" - def int adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) echo "Adjusted change count: ${adjustedchangecount}" - def gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD") + gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD").trim() echo "Revision: ${gitrevision}" - def remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin") + remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin").trim() echo "Remote info: ${remoteinfo}" - def remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin") + remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] @@ -41,9 +45,7 @@ def getVersionPrefix(String sourcebranch) { return mastertypeversionprefix } - echo "sourcebranch is now: ${sourcebranch}" def splits2 = sourcebranch.split('\\.') - echo "Splits2 is: ${splits2}" if (splits2.size() < 1) { def thirdpartyversionprefix = "1.25000.0." echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" @@ -51,7 +53,6 @@ def getVersionPrefix(String sourcebranch) { } splits = [] for (split in splits2) { - echo "Split is: ${split}" if (split.isInteger()) { splits.add(split) } @@ -59,7 +60,6 @@ def getVersionPrefix(String sourcebranch) { return mastertypeversionprefix } } - echo "Splits is: ${splits}" def versionprefix = "" if (splits.size() < 1) { @@ -69,7 +69,6 @@ def getVersionPrefix(String sourcebranch) { else { def int splitnumbercount = 0 for (split in splits) { - echo "Split is ${split}" if (split.isInteger()) { versionprefix += split + "." splitnumbercount += 1 @@ -96,6 +95,11 @@ def getVersionPrefix(String sourcebranch) { } String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) def versionprefix = getVersionPrefix(sourcerevision) From fa26214f9e65cde90568843a5cf683cb935252d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:26:40 +0000 Subject: [PATCH 128/589] Improve meta gathering --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 3a2a408..50a1655 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -39,7 +39,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 838460a826e7ad122a507e641260b736e7149b0b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:30:56 +0000 Subject: [PATCH 129/589] Improve meta gathering --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 50a1655..bbe7bb9 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -39,7 +39,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now.toString()) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 6ece33765635a185bd017e2eb1e18797379bbe13 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:40:37 +0000 Subject: [PATCH 130/589] Improve meta gathering --- versionnumbering/generate.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3c475bc..16fb1a9 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -7,18 +7,18 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count").trim() + def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) echo "Adjusted change count: ${adjustedchangecount}" - gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD").trim() + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() echo "Revision: ${gitrevision}" - remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin").trim() + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 9e512c2774e23037348c63eca63122313409d110 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 11:32:04 +0000 Subject: [PATCH 131/589] Improve meta gathering --- differences/do.groovy | 10 ++++------ meta/folder.groovy | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 9e572af..35e0980 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,10 +1,8 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" -DIFFERENCEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/differences" - -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEFOLDER}") { - cs = autobuilder.commonscripts() +def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { + def String cs = autobuilder.commonscripts() + def String difffolder = meta.folder() + def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" diff --git a/meta/folder.groovy b/meta/folder.groovy index 045d67d..25d5f1f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,7 +1,4 @@ -METAFOLDERNAME = "meta" -env.METAARTIFACTS = "${METAFOLDERNAME}/**" - abstract class GISJsonSerialised implements Serializable { def script @@ -20,7 +17,7 @@ abstract class GISJsonSerialised implements Serializable { class ProductInfo extends GISJsonSerialised { private String productname private String subcomponentname - private String jirakey + String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { super(script) @@ -191,6 +188,10 @@ class ConfigurationInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { + private String METAFOLDERNAME = "meta" + private String METAFILENAME = "info.json" + private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private String format = '0.1' ProductInfo productinfo GitInfo gitinfo @@ -250,17 +251,26 @@ class MetaInfo extends GISJsonSerialised { return data } - def setproductinfo(productname, subcomponentname, jirakey) { - productinfo.set(productname, subcomponentname, jirakey) + + String folder() { + return METAFOLDERNAME + } + + String generateartifacts() { + def String metafilepath = "${METAFOLDERNAME}/${METAFILENAME}" + saveJsonSerialised(metafilepath) + return METAARTIFACTS } } MetaInfo metareset(script) { echo "Resetting meta folder" - dir("${WORKSPACE}/${METAFOLDERNAME}") { + def mi = new MetaInfo(script) + + dir(mi.folder()) { deleteDir() } - return new MetaInfo(script) + return mi } return this; From bebea3f7f7fd9d520ba415cadb6422bee24d6f5a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 13:15:10 +0000 Subject: [PATCH 132/589] Add checksum info to meta --- checksum/utils.groovy | 15 +++----- meta/folder.groovy | 83 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 748472f..c090bf3 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,21 +1,14 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" -CHECKSUMFOLDER = "${metafolder.METAFOLDERNAME}/checksums" - // Generate a checksum file of from a list of files/directories relative to workspace -def generatelist(checksummable) { - load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - sums = "" +def generatelist(meta, checksummable) { for (filepath in checksummable) { - echo "Checksumming ${filepath}" + def checksumset = meta.checksuminfo.addchecksumset(filepath) def files = findFiles(glob: filepath) for (file in files) { - def sum = sha1(file: file.path) - sums += "${sum} ${file.path}${nl}" // Format suitable for sha1sum utility + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) } } - writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 25d5f1f..152121c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -187,6 +187,86 @@ class ConfigurationInfo extends GISJsonSerialised { } } +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath, String sum) { + super(checksumset.script) + this.filepath = filepath + this.sum = sum + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String artifactpath + private ArrayList checksumpairs + + ChecksumInfo(script, artifactpath) { + super(script) + this.artifactpath = artifactpath + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } + + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" @@ -198,6 +278,7 @@ class MetaInfo extends GISJsonSerialised { JenkinsBuildInfo jenkinsbuildinfo BuildStampInfo buildstampinfo ReleaseInfo releaseinfo + ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList associatedmeta @@ -208,6 +289,7 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo = new JenkinsBuildInfo(script) buildstampinfo = new BuildStampInfo(script) releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() associatedmeta = new ArrayList() } @@ -246,6 +328,7 @@ class MetaInfo extends GISJsonSerialised { 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), 'associatedmeta' : associatedmetaserialised(), ] return data From 2b6378e8e41fe7ea26ead77643be8db2b5e8a761 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 18:33:55 +0000 Subject: [PATCH 133/589] Add checksum info to meta --- meta/folder.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 152121c..9a89b70 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -192,7 +192,7 @@ class ChecksumPair extends GISJsonSerialised { private String sum ChecksumPair(script, String filepath, String sum) { - super(checksumset.script) + super(script) this.filepath = filepath this.sum = sum } @@ -210,9 +210,10 @@ class ChecksumSet extends GISJsonSerialised { private String artifactpath private ArrayList checksumpairs - ChecksumInfo(script, artifactpath) { + ChecksumSet(script, artifactpath) { super(script) this.artifactpath = artifactpath + checksumpairs = new ArrayList() } ChecksumPair addChecksumPair(String filepath, String sum) { @@ -236,17 +237,20 @@ class ChecksumSet extends GISJsonSerialised { ] return data } - +} class ChecksumInfo extends GISJsonSerialised { private ArrayList checksumsets ChecksumInfo(script) { super(script) + checksumsets = new ArrayList() } ChecksumSet addChecksumSet(String artifactpath) { + script.echo "Entering addChecksumSet" def cs = new ChecksumSet(script, artifactpath) + script.echo "Created ChecksumSet" checksumsets.add(cs) return cs } @@ -339,8 +343,7 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts() { - def String metafilepath = "${METAFOLDERNAME}/${METAFILENAME}" + String generateartifacts(String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}") { saveJsonSerialised(metafilepath) return METAARTIFACTS } From 084e8367d7b1907771e65a9f1689686ca605fdc5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 18:40:12 +0000 Subject: [PATCH 134/589] Add checksum info to meta --- checksum/utils.groovy | 4 ++-- meta/folder.groovy | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index c090bf3..c2503cd 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -2,7 +2,7 @@ // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(meta, checksummable) { for (filepath in checksummable) { - def checksumset = meta.checksuminfo.addchecksumset(filepath) + def checksumset = meta.checksuminfo.addChecksumSet(filepath) def files = findFiles(glob: filepath) for (file in files) { def sum = sha1(file: file.path).trim() @@ -11,4 +11,4 @@ def generatelist(meta, checksummable) { } } -return this; +return this; \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy index 9a89b70..421fec8 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -248,9 +248,7 @@ class ChecksumInfo extends GISJsonSerialised { } ChecksumSet addChecksumSet(String artifactpath) { - script.echo "Entering addChecksumSet" def cs = new ChecksumSet(script, artifactpath) - script.echo "Created ChecksumSet" checksumsets.add(cs) return cs } From 8d90a54ebd266a26ed229d56bd5c741e054ad55e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 20:19:08 +0000 Subject: [PATCH 135/589] Add publishable items to meta --- meta/folder.groovy | 32 ++++++++++++++++++++++++++++++++ publishing/utils.groovy | 20 ++------------------ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 421fec8..7471f74 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -269,6 +269,22 @@ class ChecksumInfo extends GISJsonSerialised { } } +class PublishableInfo extends GISJsonSerialised { + private String filepath + + PublishableInfo(script, String filepath = "") { + super(script) + this.filepath = filepath + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" @@ -282,6 +298,7 @@ class MetaInfo extends GISJsonSerialised { ReleaseInfo releaseinfo ChecksumInfo checksuminfo private ArrayList configurationinfo + private ArrayList publishableinfo private ArrayList associatedmeta MetaInfo(script) { @@ -293,6 +310,7 @@ class MetaInfo extends GISJsonSerialised { releaseinfo = new ReleaseInfo(script) checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() + publishableinfo = new ArrayList() associatedmeta = new ArrayList() } @@ -304,11 +322,24 @@ class MetaInfo extends GISJsonSerialised { return configuration } + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + publishableinfo.add(pi) + } + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) configurationinfo.add(ci) } + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { @@ -331,6 +362,7 @@ class MetaInfo extends GISJsonSerialised { 'releaseinfo' : releaseinfo.serialisationdata(), 'configuration' : configurationinfoserialised(), 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), 'associatedmeta' : associatedmetaserialised(), ] return data diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 72c6566..145a7e9 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,26 +1,10 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" - // Generate a list of artifact files that are publishable to outside parties -def generatelist(publishable) { - def PUBLISHABLEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/publishable" - sh """ - rm -rf "${PUBLISHABLEFOLDER}" - mkdir -p "${PUBLISHABLEFOLDER}" - """ +def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { - def files = findFiles(glob: filepath) - for (file in files) { - if (!file.directory) { - filelist.add(file.path) - } - } + meta.addPublishable(filepath) } - def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) - def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" - writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) } return this; From a06e33d10d6690247fa569a7331eb376ebfb4ab3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 22:17:38 +0000 Subject: [PATCH 136/589] Add release information items to meta --- meta/folder.groovy | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7471f74..b5cc379 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -143,20 +143,35 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private String releasejobname + private String releaserepoURL + private ArrayList releaseitems - ReleaseInfo(script, releasejobname = "") { + ReleaseInfo(script, releaserepoURL = "") { super(script) - this.releasejobname = releasejobname + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() } - void set(String releasejobname) { - this.releasejobname = releasejobname + void set(String releaserepoURL = "") { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + releaseitems.add(releaseitem) + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items } LinkedHashMap serialisationdata() { def data = [ - 'releasejobname' : releasejobname, + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), ] return data } @@ -373,9 +388,21 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}") { + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), + ArrayList releasedfromsubmeta = new ArrayList(), + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = METAARTIFACTS + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } saveJsonSerialised(metafilepath) - return METAARTIFACTS + return artifactlist } } From f14ce9fc0cf3d85cbc66a8f4155515c538dd603c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 17:23:17 +0000 Subject: [PATCH 137/589] Deserialise meta from file and associate with job meta --- import/importer.groovy | 14 ++++ meta/folder.groovy | 154 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 import/importer.groovy diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..ba912f4 --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,14 @@ + +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: buildselector + ], + target: destpath + ) + meta.importrelated(destpath) +} \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy index b5cc379..9d90cee 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -6,7 +6,9 @@ abstract class GISJsonSerialised implements Serializable { this.script = script } - abstract LinkedHashMap serialisationdata(); + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -32,6 +34,12 @@ class ProductInfo extends GISJsonSerialised { this.jirakey = jirakey } + void deserialiseJson(rawmeta) { + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + jirakey = rawmeta['jirakey'] + } + LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, @@ -66,6 +74,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + void deserialiseJson(rawmeta) { + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, @@ -99,6 +115,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } + void deserialiseJson(rawmeta) { + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + } + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, @@ -131,6 +154,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + void deserialiseJson(rawmeta) { + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + buildstampcommasep = rawmeta['buildstampcommasep'] + } + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, @@ -168,6 +198,14 @@ class ReleaseInfo extends GISJsonSerialised { return items } + void deserialiseJson(rawmeta) { + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + } + LinkedHashMap serialisationdata() { def data = [ 'releaserepoURL' : releaserepoURL, @@ -191,6 +229,13 @@ class ConfigurationInfo extends GISJsonSerialised { this.configurationSHA = configurationSHA } + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + LinkedHashMap serialisationdata() { def data = [ 'configurationname' : configurationname, @@ -212,6 +257,11 @@ class ChecksumPair extends GISJsonSerialised { this.sum = sum } + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + LinkedHashMap serialisationdata() { def data = [ 'filepath' : filepath, @@ -245,6 +295,16 @@ class ChecksumSet extends GISJsonSerialised { return associated } + void deserialiseJson(rawmeta) { + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + LinkedHashMap serialisationdata() { def data = [ 'artifactpath' : artifactpath, @@ -276,6 +336,15 @@ class ChecksumInfo extends GISJsonSerialised { return associated } + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + LinkedHashMap serialisationdata() { def data = [ 'checksumsets' : checksumsetsserialised(), @@ -292,6 +361,10 @@ class PublishableInfo extends GISJsonSerialised { this.filepath = filepath } + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + } + LinkedHashMap serialisationdata() { def data = [ 'filepath' : filepath, @@ -301,11 +374,37 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { + private String FORMAT = "0.1" private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" private String METAARTIFACTS = "${METAFOLDERNAME}/**" - private String format = '0.1' + class RelatedMetaInfo extends GISJsonSerialised { + String offsetpath + MetaInfo meta + + RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + super(script) + this.offsetpath = offsetpath + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'offsetpath' : offsetpath, + 'meta' : meta.serialisationdata(), + ] + return data + } + } + + private String format = FORMAT ProductInfo productinfo GitInfo gitinfo JenkinsBuildInfo jenkinsbuildinfo @@ -314,7 +413,7 @@ class MetaInfo extends GISJsonSerialised { ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo - private ArrayList associatedmeta + private ArrayList associatedmeta MetaInfo(script) { super(script) @@ -326,7 +425,7 @@ class MetaInfo extends GISJsonSerialised { checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() publishableinfo = new ArrayList() - associatedmeta = new ArrayList() + associatedmeta = new ArrayList() } List configurationinfoserialised() { @@ -363,8 +462,51 @@ class MetaInfo extends GISJsonSerialised { return associated } - void addAssociatedMetaInfo(mi) { - associatedmeta.add(mi) + void deserialiseJson(rawmeta) { + if (rawmeta['format'] != FORMAT) { + script.error("Unsupported meta format found at ${filepath}") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + for (rawrelatedmeta in rawmeta['associatedmeta']) { + def rm = new RelatedMetaInfo(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + + def deserialiseJson(String filepath) { + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + MetaInfo importrelated(String destpath) { + MetaInfo mi = new MetaInfo(script) + + def importedmetafile = destpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + } + + RelatedMetaInfo rmi = new RelatedMetaInfo(script, destpath, mi) + associatedmeta.add(rmi) + + return mi } LinkedHashMap serialisationdata() { From 53878daf609b8eacf1dd815b89cc4d1e8507d5a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 18:51:51 +0000 Subject: [PATCH 138/589] Deserialise meta from file and associate with job meta --- import/importer.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index ba912f4..d3c97e4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -11,4 +11,6 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) meta.importrelated(destpath) -} \ No newline at end of file +} + +return this; \ No newline at end of file From 929c18f5b690313da022d54bd815b8b0aab76dc8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 19:54:32 +0000 Subject: [PATCH 139/589] Store offset paths relative to WORKSPACE if given as absolute paths --- meta/folder.groovy | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9d90cee..28fe85f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,3 +1,5 @@ +import java.nio.file.Paths +import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { def script @@ -494,16 +496,27 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } + + String relativize(String path) { + def Path thispath = Paths.get(path) + if (thispath.startsWith(script.WORKSPACE)) { + def Path wspath = Paths.get(script.WORKSPACE) + def Path wsrelativePath = wspath.relativize(thispath) + return wsrelativePath.toString() + } + return path + } MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) - def importedmetafile = destpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + METAFOLDERNAME + "/" + METAFILENAME if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, destpath, mi) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) return mi From 8d119dcdbc340866d01f763a592649a445d4cd38 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 20:06:10 +0000 Subject: [PATCH 140/589] Only show SHA code of configuration --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 031094b..cb9354b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -2,7 +2,7 @@ clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { - return clh.getCommandOutput('git log -1 --format=oneline') + return clh.getCommandOutput('git rev-parse HEAD') } } From 9b51bd7cf89adcbd1de56613d51d22336bec760c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 21:05:06 +0000 Subject: [PATCH 141/589] Handle more version number strings --- versionnumbering/generate.groovy | 46 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 16fb1a9..59fe2a3 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -28,6 +28,8 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { + def int MAXVERSIONNUMBER = 65535 + echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -51,43 +53,45 @@ def getVersionPrefix(String sourcebranch) { echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" return thirdpartyversionprefix } - splits = [] + for (split in splits2) { - if (split.isInteger()) { - splits.add(split) - } - else if (split == "x") { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" return mastertypeversionprefix } } + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + int integer = integerstring.toInteger() + integer = integer % (MAXVERSIONNUMBER + 1) + integers.add(integer) + } + def versionprefix = "" - if (splits.size() < 1) { + if (integers.size() < 1) { echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" versionprefix = defaultversionprefix } else { - def int splitnumbercount = 0 - for (split in splits) { - if (split.isInteger()) { - versionprefix += split + "." - splitnumbercount += 1 - } - else if (splitnumbercount > 2) { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { break } } - if (splitnumbercount == 0) { + if (integernumbercount == 0) { echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" versionprefix = defaultversionprefix } - else { - if (splitnumbercount == 1) { - versionprefix += "0.0." - } - else if (splitnumbercount == 2) { - versionprefix += "0." - } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." } } echo "Determined version prefix to be: ${versionprefix}" From 3bad7e2597a9242c529a66b3dbb2949de572ac79 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 21:36:36 +0000 Subject: [PATCH 142/589] Parameterise the release repo --- autobuilder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index bbe7bb9..732e268 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -40,7 +40,9 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - + + meta.releaseinfo.set(releaserepo) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 3eb169c9da50d78f63ac92c33a7a8fd585049ef0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:20:04 +0000 Subject: [PATCH 143/589] Check for existence of release repo by checking it out --- autobuilder.groovy | 3 ++- release/registerrepo.groovy | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 release/registerrepo.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 732e268..2a04925 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -41,7 +41,8 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - meta.releaseinfo.set(releaserepo) + def rr = load("${commonscripts()}/release/registerrepo.groovy") + rr.registerrepo(meta, releaserepo) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..1fc76ab --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo) { + if (releaserepo?.trim()) { + vc.gitCheckout(releaserepo, "origin/master", "meta/releaserepo") + } + meta.releaseinfo.set(releaserepo) +} + +return this; From d297db525283febc1e133fb294a0a66b94e1d372 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:32:50 +0000 Subject: [PATCH 144/589] Save the temp repo load in a tmp folder --- release/registerrepo.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index 1fc76ab..ff4d85d 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -4,7 +4,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data def registerrepo(meta, String releaserepo) { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "meta/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") } meta.releaseinfo.set(releaserepo) } From 0a7a6673bcb7f21a84fb37fb85f15aa846c2591e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:59:38 +0000 Subject: [PATCH 145/589] Make compile job use meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 28fe85f..0f9e0d5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -138,8 +138,8 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix private int revision - private String buildstamp - private String buildstampcommasep + String buildstamp + String buildstampcommasep BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { super(script) From c9a8032b36143cd19a38416561047a8948afbd6e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 11:24:37 +0000 Subject: [PATCH 146/589] Rename compile_init --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0f9e0d5..189014c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -162,7 +162,7 @@ class BuildStampInfo extends GISJsonSerialised { buildstamp = rawmeta['buildstamp'] buildstampcommasep = rawmeta['buildstampcommasep'] } - + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, From b5d80d67e8e3f561ca4e4c56db73cfc84fcce77d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 12:34:53 +0000 Subject: [PATCH 147/589] Correct checksumset deserialisation --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 189014c..e9491fe 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -253,7 +253,7 @@ class ChecksumPair extends GISJsonSerialised { private String filepath private String sum - ChecksumPair(script, String filepath, String sum) { + ChecksumPair(script, String filepath = "", String sum = "") { super(script) this.filepath = filepath this.sum = sum @@ -277,7 +277,7 @@ class ChecksumSet extends GISJsonSerialised { private String artifactpath private ArrayList checksumpairs - ChecksumSet(script, artifactpath) { + ChecksumSet(script, artifactpath = "") { super(script) this.artifactpath = artifactpath checksumpairs = new ArrayList() From 3752aa82ca371a5e4f278fb1821587265e580cf8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 13:18:35 +0000 Subject: [PATCH 148/589] Move image create job to use meta --- meta/folder.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index e9491fe..8f26265 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -507,11 +507,15 @@ class MetaInfo extends GISJsonSerialised { return path } + String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) def String adjustedpath = relativize(destpath) - def importedmetafile = adjustedpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) } From f50bbb36abbcc4e49af0c919afa51b706970e4dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 14:14:13 +0000 Subject: [PATCH 149/589] Update build job to meta --- artifacts/metaexport.groovy | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 66a8153..0cfc773 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,21 +1,19 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(subJobs) { - artifactFolders = "" - for (subJob in subJobs) { - subJobObject = subJob[0] - artifactFolder = subJob[1] - if (subJobObject != null) { - subJobName = subJobObject.getProjectName() - subJobNumber = subJobObject.getNumber() - destFolder = "${artifactFolder}" +def combineSubjobArtifacts(meta, subJobs) { + def im = load("${commonscripts()}/import/importer.groovy") - dir("${destFolder}") { + def artifactFolders = "" + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getProjectName() + def subJobNumber = subJobObject.getNumber() + destFolder = subJob[1] + dir(destFolder) { deleteDir() } - - echo "Copying artifacts from: Job: ${subJobName} Number: ${subJobNumber} to folder: ${destFolder}" - copyArtifacts fingerprintArtifacts: true, projectName: "${subJobName}", selector: specific("${subJobNumber}"), target: "${destFolder}" + im.artifactimport(meta, subJobName, specific(subJobNumber), destFolder) artifactFolders += "${destFolder}/**," } } From ee97d816c8cbb5de9f4224a34c48a525ebb048ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 25 Nov 2018 17:01:32 +0000 Subject: [PATCH 150/589] Update build job to meta --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0cfc773..016add9 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,7 +1,7 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, subJobs) { - def im = load("${commonscripts()}/import/importer.groovy") + def im = load("${autobuilder.commonscripts()}/import/importer.groovy") def artifactFolders = "" for (subJob in subJobs) { From 36b2d8bd90195a63d8864c1deba918e632fec38c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 25 Nov 2018 23:19:27 +0000 Subject: [PATCH 151/589] Make release job use meta --- artifacts/metaexport.groovy | 4 ++-- import/importer.groovy | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 016add9..62a6ba4 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,7 +1,7 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, subJobs) { - def im = load("${autobuilder.commonscripts()}/import/importer.groovy") + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def artifactFolders = "" for (subJob in subJobs) { @@ -13,7 +13,7 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - im.artifactimport(meta, subJobName, specific(subJobNumber), destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders += "${destFolder}/**," } } diff --git a/import/importer.groovy b/import/importer.groovy index d3c97e4..a9e28b2 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,15 @@ +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: specific(buildnumber.toString()), + target: destpath + ) + meta.importrelated(destpath) +} + def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -13,4 +24,4 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat meta.importrelated(destpath) } -return this; \ No newline at end of file +return this; From 9106a00c932747ee5fd0a6baf4cf95d811b60c71 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 08:29:56 +0000 Subject: [PATCH 152/589] Build up list of artifacts --- artifacts/metaexport.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 62a6ba4..cf5e9b3 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -3,7 +3,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def artifactFolders = "" + def artifactFolders = [] for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,7 +14,7 @@ def combineSubjobArtifacts(meta, subJobs) { deleteDir() } importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - artifactFolders += "${destFolder}/**," + artifactFolders.add("${destFolder}/**") } } return artifactFolders; From 0150bf139f8e29a8816c786a4ea01f8bce10ecf6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 11:16:53 +0000 Subject: [PATCH 153/589] Explicitly return artifact list as ArrayList --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index cf5e9b3..b6a832b 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -3,7 +3,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def artifactFolders = [] + def ArrayList artifactFolders = new ArrayList() for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { From 7e58b0b5a1babecf042aafc4e79651b8819658d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 18:17:55 +0000 Subject: [PATCH 154/589] Walk meta data for release info --- import/importer.groovy | 4 ++-- meta/folder.groovy | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index a9e28b2..4d24d30 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -7,7 +7,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +21,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f26265..c2c30a4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -175,8 +175,8 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private String releaserepoURL - private ArrayList releaseitems + String releaserepoURL + ArrayList releaseitems ReleaseInfo(script, releaserepoURL = "") { super(script) @@ -563,6 +563,26 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } + + // Recurse tree of meta looking for release info and accumulate path + private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { + if (releaseinfo.releaserepoURL) { + ArrayList rmi(script, pathsofar, this) + releasemetacollection.add(rmi) + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath + releasemetacollection = releasewalkerp(releasemetacollection, newpathsofar) + } + return releasemetacollection + } + + ArrayList releasewalker(String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(releasemetacollection, startpath) + return releasemetacollection + } } MetaInfo metareset(script) { From 1dffa1ef60eb6f9b4da17597dfee0d6992e4b612 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 18:37:54 +0000 Subject: [PATCH 155/589] Walk meta data for release info --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index c2c30a4..361c120 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -567,7 +567,7 @@ class MetaInfo extends GISJsonSerialised { // Recurse tree of meta looking for release info and accumulate path private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { if (releaseinfo.releaserepoURL) { - ArrayList rmi(script, pathsofar, this) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) releasemetacollection.add(rmi) } for (relatedmetainfo in associatedmeta) { From 68f24e6a30a3f8330a6e5914bb99bd69fc153fbf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 21:18:12 +0000 Subject: [PATCH 156/589] Walk meta data for release info --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 361c120..b204462 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -573,7 +573,7 @@ class MetaInfo extends GISJsonSerialised { for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath - releasemetacollection = releasewalkerp(releasemetacollection, newpathsofar) + releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } return releasemetacollection } From 84589b75474c3189c35c18320fc13da0aff4649f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 08:34:19 +0000 Subject: [PATCH 157/589] Correct release path --- meta/folder.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index b204462..88c5e17 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -572,7 +572,10 @@ class MetaInfo extends GISJsonSerialised { } for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath + if (pathsofar) { + pathsofar += "/" + } + String newpathsofar = pathsofar + relatedmetainfo.offsetpath releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } return releasemetacollection From 6211375100f8e0f029d653d1e7d3851bbe19261f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 08:51:27 +0000 Subject: [PATCH 158/589] Correct release path --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 88c5e17..802443d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -570,11 +570,11 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) releasemetacollection.add(rmi) } + if (pathsofar) { + pathsofar += "/" + } for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta - if (pathsofar) { - pathsofar += "/" - } String newpathsofar = pathsofar + relatedmetainfo.offsetpath releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } From da2e95a8705d2ead0a4e2c9b5f05dd592345226c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 16:52:38 +0000 Subject: [PATCH 159/589] Check out release repos --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 802443d..f5e7c93 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,7 +564,7 @@ class MetaInfo extends GISJsonSerialised { return artifactlist } - // Recurse tree of meta looking for release info and accumulate path + // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { if (releaseinfo.releaserepoURL) { RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) From 976994accc03b3374a4de0efa06fa0b438ea5b3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 18:09:27 +0000 Subject: [PATCH 160/589] Check out release repos --- meta/folder.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f5e7c93..1e45319 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -507,7 +507,7 @@ class MetaInfo extends GISJsonSerialised { return path } - String metafilepath() { + static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } @@ -598,4 +598,9 @@ MetaInfo metareset(script) { return mi } +MetaInfo metafromfolder(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") +} + return this; From 03921f73d4226056021efb821daaf18eec6445dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 18:47:42 +0000 Subject: [PATCH 161/589] Check out release repos --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1e45319..de445f1 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -376,10 +376,10 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private String FORMAT = "0.1" - private String METAFOLDERNAME = "meta" - private String METAFILENAME = "info.json" - private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String FORMAT = "0.1" + private static String METAFOLDERNAME = "meta" + private static String METAFILENAME = "info.json" + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath From 0ee39e5c95e1b9b2d1beba62b40548bb4258ea7e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 19:10:54 +0000 Subject: [PATCH 162/589] Check out release repos --- autobuilder.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 2a04925..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -25,17 +27,21 @@ def getautobuildpath() { return autobuildroot } +def getmetafolderscript() { + return metafolderscript +} + // Start up boilderplate def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - def meta = metafolder.metareset(script) + def meta = getmetafolderscript().metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) From 6a08d9ef26061b9c6d97afa2a3597aa3acd69c89 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:19:18 +0000 Subject: [PATCH 163/589] Check out release repos --- autobuilder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..885d243 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,6 +28,7 @@ def getautobuildpath() { } def getmetafolderscript() { + echo "metafolderscript is type: ${metafolderscript.toClass()}" return metafolderscript } From 3c8510bb275a21aa1a883eca2dcc99c9c0b49f7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:23:17 +0000 Subject: [PATCH 164/589] Check out release repos --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 885d243..e40a67e 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,7 +28,7 @@ def getautobuildpath() { } def getmetafolderscript() { - echo "metafolderscript is type: ${metafolderscript.toClass()}" + echo "metafolderscript is type: ${metafolderscript.getClass()}" return metafolderscript } From e7085c9e219608078132ee5b65423a2c0f0f401c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:41:01 +0000 Subject: [PATCH 165/589] Check out release repos --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index de445f1..458156d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -601,6 +601,7 @@ MetaInfo metareset(script) { MetaInfo metafromfolder(String folder = "") { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta } return this; From fca9f9f00d48ff56eb76fcbeeb27914af5fe1246 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 22:35:37 +0000 Subject: [PATCH 166/589] Remove debug --- autobuilder.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index e40a67e..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,7 +28,6 @@ def getautobuildpath() { } def getmetafolderscript() { - echo "metafolderscript is type: ${metafolderscript.getClass()}" return metafolderscript } From 3f8bb3f6f7c7de23558eaa4111dc8de2c1a33fcc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 09:07:06 +0000 Subject: [PATCH 167/589] Create a version number for the all build --- artifacts/metaexport.groovy | 8 ++++++-- meta/folder.groovy | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b6a832b..e87e3d9 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,6 +4,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() + int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -13,11 +14,14 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + revisiontotal += importedmeta.buildstampinfo.revision artifactFolders.add("${destFolder}/**") } } - return artifactFolders; + def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + revisiontotal = revisiontotal % (versiongenerator.MAXVERSIONNUMBER + 1) + return [artifactFolders, revisiontotal] } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 458156d..74ac7c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -137,7 +137,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix - private int revision + int revision String buildstamp String buildstampcommasep @@ -156,6 +156,14 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + void changerevision(int newrevision) { + revision = newrevision + buildstampsplit = buildstamp.split('.') + buildstampsplit[3] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + buildstampcommasep = buildstampsplit.join(',') + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From 1a4666fdcce33377839c1992fb2f6410a2cd877a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 11:37:10 +0000 Subject: [PATCH 168/589] Correct scope of MAXVERSIONNUMBER --- versionnumbering/generate.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 59fe2a3..cf21fd0 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,3 +1,4 @@ +int MAXVERSIONNUMBER = 65535 def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount @@ -5,8 +6,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) @@ -28,8 +27,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { - def int MAXVERSIONNUMBER = 65535 - echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() From 46ddde1b777ab67847f7ad5de4cab37ca6488143 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 13:10:26 +0000 Subject: [PATCH 169/589] Correct scope of MAXVERSIONNUMBER --- artifacts/metaexport.groovy | 2 +- versionnumbering/generate.groovy | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index e87e3d9..b4d4a6d 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -20,7 +20,7 @@ def combineSubjobArtifacts(meta, subJobs) { } } def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.MAXVERSIONNUMBER + 1) + revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) return [artifactFolders, revisiontotal] } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cf21fd0..9df06d8 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,4 +1,8 @@ -int MAXVERSIONNUMBER = 65535 + +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount @@ -8,7 +12,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -62,7 +66,7 @@ def getVersionPrefix(String sourcebranch) { def integers = [] for (integerstring in integerstrings) { int integer = integerstring.toInteger() - integer = integer % (MAXVERSIONNUMBER + 1) + integer = integer % (maxversionnumber() + 1) integers.add(integer) } From 58b6100215be6ed6ddbe3eb7abacaacac88fff50 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:11:08 +0000 Subject: [PATCH 170/589] Load in artifacts for release --- meta/folder.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 74ac7c4..822472d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -594,6 +594,17 @@ class MetaInfo extends GISJsonSerialised { releasemetacollection = releasewalkerp(releasemetacollection, startpath) return releasemetacollection } + + Map releasewalkerunique(String startpath = "") { + Map releasemetauniquecollection = new Map() + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(releasemetacollection, startpath) + for (relatedmetainfo in releasemetacollection) { + String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetauniquecollection + } } MetaInfo metareset(script) { From c095a2c19ef6afcb69febebfb1a5b5e20392a9d3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:17:09 +0000 Subject: [PATCH 171/589] Load in artifacts for release --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 822472d..8f8ea0a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -596,7 +596,7 @@ class MetaInfo extends GISJsonSerialised { } Map releasewalkerunique(String startpath = "") { - Map releasemetauniquecollection = new Map() + Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { From 911d147f81b3c3a3712490e8aca945e96b6c2118 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:40:47 +0000 Subject: [PATCH 172/589] Load in artifacts for release --- meta/folder.groovy | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f8ea0a..24bcbf5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -573,32 +573,34 @@ class MetaInfo extends GISJsonSerialised { } // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java - private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { - if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) - releasemetacollection.add(rmi) - } - if (pathsofar) { - pathsofar += "/" - } - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + relatedmetainfo.offsetpath - releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) + private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { + if (maxdepth > 0) { + if (releaseinfo.releaserepoURL) { + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + releasemetacollection.add(rmi) + } + if (pathsofar) { + pathsofar += "/" + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + relatedmetainfo.offsetpath + releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) + } } return releasemetacollection } - ArrayList releasewalker(String startpath = "") { + ArrayList releasewalker(int maxdepth, String startpath = "") { ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(releasemetacollection, startpath) + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) return releasemetacollection } - Map releasewalkerunique(String startpath = "") { + Map releasewalkerunique(int maxdepth, String startpath = "") { Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(releasemetacollection, startpath) + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname releasemetauniquecollection.put(uniquestring, relatedmetainfo) From 1acfec14d58fccd275ea435ef4bae6423aa5dde1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 19:24:10 +0000 Subject: [PATCH 173/589] Perform git push operations --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 24bcbf5..b8120a6 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -158,7 +158,7 @@ class BuildStampInfo extends GISJsonSerialised { void changerevision(int newrevision) { revision = newrevision - buildstampsplit = buildstamp.split('.') + def List buildstampsplit = buildstamp.split('.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') From a43f93cc15a854ff0425b36a355d6acaa5b614e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 19:44:11 +0000 Subject: [PATCH 174/589] Perform git push operations --- meta/folder.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index b8120a6..cfa0a6e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,11 +157,13 @@ class BuildStampInfo extends GISJsonSerialised { } void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") revision = newrevision def List buildstampsplit = buildstamp.split('.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') + script.echo("Build stamp now ${buildstamp}") } void deserialiseJson(rawmeta) { From 95b1a75e0c8c32161cac0ff241e32f0d9aedf5c7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 00:33:39 +0000 Subject: [PATCH 175/589] Calculate version number earlier --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cfa0a6e..ed17a91 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -159,7 +159,7 @@ class BuildStampInfo extends GISJsonSerialised { void changerevision(int newrevision) { script.echo("Build stamp currently ${buildstamp}") revision = newrevision - def List buildstampsplit = buildstamp.split('.') + def List buildstampsplit = buildstamp.split('\\.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') From 24b138ac8ffa91bc3bfee0403212417f8cb0b1de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:06:46 +0000 Subject: [PATCH 176/589] Remote URL fetch was not working --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ed17a91..2640567 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,4 +1,4 @@ -import java.nio.file.Paths +urimport java.nio.file.Paths import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 9df06d8..f185c40 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -21,7 +21,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 3fceb3282905b6cef5137eacf9cb0a17774e1d47 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:10:04 +0000 Subject: [PATCH 177/589] Remote URL fetch was not working --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2640567..ed17a91 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,4 +1,4 @@ -urimport java.nio.file.Paths +import java.nio.file.Paths import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { From 347e544b459cb5206c0145bc3007288412a062f3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:36:20 +0000 Subject: [PATCH 178/589] Remove comma separated buildstamp store --- meta/folder.groovy | 17 ++++++++--------- versionnumbering/generate.groovy | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ed17a91..76fbe43 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -139,30 +139,31 @@ class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix int revision String buildstamp - String buildstampcommasep - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } - void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } + String buildstampcommasep() { + buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + void changerevision(int newrevision) { script.echo("Build stamp currently ${buildstamp}") revision = newrevision def List buildstampsplit = buildstamp.split('\\.') - buildstampsplit[3] = newrevision.toString() + buildstampsplit[-1] = newrevision.toString() buildstamp = buildstampsplit.join('.') - buildstampcommasep = buildstampsplit.join(',') script.echo("Build stamp now ${buildstamp}") } @@ -170,7 +171,6 @@ class BuildStampInfo extends GISJsonSerialised { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] - buildstampcommasep = rawmeta['buildstampcommasep'] } LinkedHashMap serialisationdata() { @@ -178,7 +178,6 @@ class BuildStampInfo extends GISJsonSerialised { 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, - 'buildstampcommasep' : buildstampcommasep, ] return data } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f185c40..a7b583a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -113,7 +113,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum echo "Build stamp is: ${buildstamp}" meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) return buildstamp } From 574c2155a776b67f164ab426da7fcbe606a333f8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 12:48:41 +0000 Subject: [PATCH 179/589] Correct buildstamp comma sep generation --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 76fbe43..570a3ea 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -154,7 +154,7 @@ class BuildStampInfo extends GISJsonSerialised { } String buildstampcommasep() { - buildstampcommasep = buildstamp.replace(".", ",") + def String buildstampcommasep = buildstamp.replace(".", ",") return buildstampcommasep } From d33e50f3cc7779f41f6703b52ef97c8e73e62b78 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 13:23:40 +0000 Subject: [PATCH 180/589] Remove redundant script; Correct artifact export of release job --- versionnumbering/metaversion.groovy | 100 ---------------------------- 1 file changed, 100 deletions(-) delete mode 100644 versionnumbering/metaversion.groovy diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy deleted file mode 100644 index 44bfbde..0000000 --- a/versionnumbering/metaversion.groovy +++ /dev/null @@ -1,100 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - -def prefixline(rootpath, filepath) { - def fileContents = readFile("${filepath}").trim() - def fileContentsNormalized = fileContents.normalize() - - def newFileContents = "" - def lines = fileContentsNormalized.readLines() - for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + "${nl}" - } - return newFileContents -} - -def mergemeta(rootpath, versionFolder, outputFolder) { - echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents - - def intrevision = 0 - if (fileExists("${versionFolder}/revision.txt")) { - def revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - } - else { - intrevision = 1 - } - - return intrevision -} - -def appendcomponentinfo(rootpath, versionFolder, outputFolder) { - echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents - } - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = readFile("${buildStampFile}").trim() - - def remoteurlFile = "${versionFolder}/remoteurl.txt" - def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - - def gitRevisionFile = "${versionFolder}/gitrevision.txt" - def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - def releaseJobNameFile = "${versionFolder}/releasejobname.txt" - def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - def revision = readFile("${versionFolder}/revision.txt").trim() - def intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - - return intrevision -} - -// Each version folder should have been created using version creation scripts in this Common_scripts folder -// as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - def totalrevision = 0 - dir("${outputFolder}") { - deleteDir() - } - for (artifactItem in artifactList) { - (rootpath, versionFolder) = artifactItem - echo "Inspecting folder: ${versionFolder}" - def metaversionsignaturefile = "${versionFolder}/metaversion.txt" - // If the sub project is a meta job - def intrevision = 0 - if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta(rootpath, versionFolder, outputFolder) - } - else { - intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) - } - totalrevision += intrevision - } - echo "Accumulated revision is: ${totalrevision}" - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) -} - -return this; \ No newline at end of file From cc99fcf2a17aff04d651b5a24338cb8240476c9c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:03:12 +0000 Subject: [PATCH 181/589] Alter differences file generation; Merge differences when importing --- differences/do.groovy | 6 +++--- differences/show_release_differences.py | 11 +++++------ meta/folder.groovy | 22 +++++++++++++++++++--- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 35e0980..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,16 +1,16 @@ def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { def String cs = autobuilder.commonscripts() - def String difffolder = meta.folder() + def String difffile = meta.difffilepath() def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0813356..fee6f7b 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -172,7 +172,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") + parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +183,13 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - difffolder = args.difffolder + difffile = args.difffile print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - if difffolder: - makeFolder(difffolder) - outfh = open(os.path.join(difffolder, "differences.txt"), 'w') + if difffile: + outfh = open(difffile), 'w') else: outfh = sys.stdout diff --git a/meta/folder.groovy b/meta/folder.groovy index 570a3ea..8bd80d3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,8 +19,8 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - private String productname - private String subcomponentname + String productname + String subcomponentname String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { @@ -388,6 +388,7 @@ class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" private static String METAFILENAME = "info.json" + private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { @@ -519,7 +520,11 @@ class MetaInfo extends GISJsonSerialised { static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } - + + static String difffilepath() { + return METAFOLDERNAME + "/" + METADIFFFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) @@ -532,6 +537,17 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) + def importedmetadifffile = adjustedpath + "/" + difffilepath() + if (script.fileExists(importedmetadifffile)) { + def String newcontents = script.readFile(file: importedmetadifffile) + def String currentcontents = "" + if (script.fileExists(difffilepath())) { + currentcontents = script.readFile(file: difffilepath()) + } + + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " mi.productinfo.componentname + ":\n" + newcontents) + } + return mi } From e452d7823e3bb03a2c30157175514cdfb5c9a389 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:05:23 +0000 Subject: [PATCH 182/589] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8bd80d3..2269a77 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " mi.productinfo.componentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.componentname + ":\n" + newcontents) } return mi From c98a09f5250d499c22f3c0b426213151cee2ce44 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:31:21 +0000 Subject: [PATCH 183/589] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2269a77..ac2ab9a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.componentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) } return mi From 0f2ae8596496efab7e0bf0448bac9ec93e494346 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:36:42 +0000 Subject: [PATCH 184/589] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ac2ab9a..9c5367e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) } return mi From 94ef8956ba6bdd56bd225850ae290e2c0771dfde Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:41:49 +0000 Subject: [PATCH 185/589] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index fee6f7b..0600753 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -189,7 +189,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: - outfh = open(difffile), 'w') + outfh = open(difffile, 'w') else: outfh = sys.stdout From 5491405110db56d84eedeb45368523c1d2793721 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:51:43 +0000 Subject: [PATCH 186/589] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0600753..ddf8c0d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,10 +29,6 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(difffolder): - print("Creating folder (if not already there): {0}".format(difffolder)) - os.makedirs(difffolder, exist_ok = True) - def printDivider(outfh, times = 1, length = 100): for _ in range(times): outfh.write("{0}\n".format("*" * length)) @@ -208,7 +204,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if difffolder: + if difffile: outfh.close() return 0 except Exception as e: From 5bb08476af818e51c87b40798049bf249bc2b6a4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 15:05:23 +0000 Subject: [PATCH 187/589] Add HMBFDGMA jira comment regular expression --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index ddf8c0d..80b636e 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -80,6 +80,9 @@ jiraprojectpattern = { 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), + 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From aa497116fd9078857227afef34bf88c184030998 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 15:19:55 +0000 Subject: [PATCH 188/589] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 80b636e..5c0f15d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -188,6 +188,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: + os.makedirs(os.path.dirname(difffile), exist_ok=True) outfh = open(difffile, 'w') else: outfh = sys.stdout From db54da7b0577f004033c5104c5f22fc10f090fdc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 17:48:05 +0000 Subject: [PATCH 189/589] Move release functionality into Common_scripts --- release/do.groovy | 131 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 release/do.groovy diff --git a/release/do.groovy b/release/do.groovy new file mode 100644 index 0000000..72c2f9f --- /dev/null +++ b/release/do.groovy @@ -0,0 +1,131 @@ +boolean markunstable = false +String unstablereason = "" + +String summaryfolder = "${WORKSPACE}/summary" +String summaryfile = "${summaryfolder}/summary.txt" +String releaserepofolder = "${WORKSPACE}/releaserepos" +String tempfolder = "${WORKSPACE}/temp" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def start() { + markunstable = false + unstablereason = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary + "\n") +} + +def checkoutreleaserepo(info, String checkoutfolder) { + def String releaserepo = info.meta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component + dir(releaserepofolder) { + sh("git clone -n --depth 1 ${releaserepo} ${checkoutfolder}") + } +} + +def getcomponentartifacts(info, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + for (artifactfilter in info.meta.releaseinfo.releaseitems) { + def filterpath = artifactfilter + if (info.offsetpath) { + filterpath = info.offsetpath + '/' + artifactfilter + } + // Copy relevant artifacts to temp folder + echo "Copying artifacts: ${filterpath} to ${tempfolder}" + copyArtifacts( + filter: filterpath, + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: tempfolder + ) + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + info.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } + summaryappend("${artifactfilter},") + } +} + +def gitrelease(String releaserepo, String buildstamp) { + summaryappend(" Git operations: ") + dir(releaserepo) { + // Add anything new + sh("git add .") + summaryappend("Add,") + // Perform git commit (if necessary) + def changesoccurred = sh(script: "git status --porcelain", returnStdout: true) + if (changesoccurred) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") + } + else { + summaryappend("No-commit,") + } + // Tag with buildstamp + sh("git tag ${buildstamp}") + summaryappend("Tag (${buildstamp}),") + if (DRY_RUN_PUSH == "true") { + sh("git push -n --tags origin master") + summaryappend("Dry-Run-Push,") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend("Push") + } + } +} + +def releasecomponent(info) { + def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${info.meta.productinfo.productname} ${info.meta.productinfo.subcomponentname}: ") + checkoutreleaserepo(info, uniquefolderforrepo) + + getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) + + def String buildstamp = info.meta.buildstampinfo.buildstamp + gitrelease(uniquefolderforrepo, buildstamp) +} + +def releasehierarchy(buildmeta, int maxdepth = MAX_META_DEPTH) { + start() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = buildmeta.releasewalkerunique(maxdepth.toInteger()) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def info = componentReleaseInfo[key] + releasecomponent(info) + } + return [markunstable, unstablereason] +} + +return this; \ No newline at end of file From 66973f0034206d128ecdbd661410425d0bd4c999 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 17:56:08 +0000 Subject: [PATCH 190/589] Move release functionality into Common_scripts --- release/do.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 72c2f9f..239cc17 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -114,11 +114,12 @@ def releasecomponent(info) { gitrelease(uniquefolderforrepo, buildstamp) } -def releasehierarchy(buildmeta, int maxdepth = MAX_META_DEPTH) { +def releasehierarchy(buildmeta, String maxdepth) { + int md = maxdepth.toInteger() start() // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = buildmeta.releasewalkerunique(maxdepth.toInteger()) + componentReleaseInfo = buildmeta.releasewalkerunique(md) // For each component for (key in componentReleaseInfo.keySet()) { From f724c2115209e3d049b3e7bc381cbf5ba5609e34 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:09:13 +0000 Subject: [PATCH 191/589] Move release functionality into Common_scripts --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 239cc17..17090fe 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -1,10 +1,10 @@ -boolean markunstable = false -String unstablereason = "" +markunstable = false +unstablereason = "" -String summaryfolder = "${WORKSPACE}/summary" -String summaryfile = "${summaryfolder}/summary.txt" -String releaserepofolder = "${WORKSPACE}/releaserepos" -String tempfolder = "${WORKSPACE}/temp" +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +releaserepofolder = "${WORKSPACE}/releaserepos" +tempfolder = "${WORKSPACE}/temp" def resetfolders() { dir(summaryfolder) { From e17df02c5cb1017411d435c7b061365ea7fd287f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:11:37 +0000 Subject: [PATCH 192/589] Move release functionality into Common_scripts --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index 17090fe..177593d 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -118,6 +118,7 @@ def releasehierarchy(buildmeta, String maxdepth) { int md = maxdepth.toInteger() start() + resetfolders() // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) From 5d53126d199b99a070b4ebbf7c58b7623ea5535a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:15:32 +0000 Subject: [PATCH 193/589] Move release functionality into Common_scripts --- release/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 177593d..1eaf2b8 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -18,7 +18,7 @@ def resetfolders() { } } -def start() { +def preconditions() { markunstable = false unstablereason = "" writeFile(file: summaryfile, text: "Release summary: \n") @@ -116,9 +116,9 @@ def releasecomponent(info) { def releasehierarchy(buildmeta, String maxdepth) { int md = maxdepth.toInteger() - start() - resetfolders() + preconditions() + // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) From 8316cf4cf71f0dba161436441509e0cd168235c3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:19:59 +0000 Subject: [PATCH 194/589] Move release functionality into Common_scripts --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 1eaf2b8..264bd53 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -70,7 +70,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { } } -def gitrelease(String releaserepo, String buildstamp) { +def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { // Add anything new @@ -88,7 +88,7 @@ def gitrelease(String releaserepo, String buildstamp) { // Tag with buildstamp sh("git tag ${buildstamp}") summaryappend("Tag (${buildstamp}),") - if (DRY_RUN_PUSH == "true") { + if (dryrunpush == "true") { sh("git push -n --tags origin master") summaryappend("Dry-Run-Push,") setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") @@ -100,7 +100,7 @@ def gitrelease(String releaserepo, String buildstamp) { } } -def releasecomponent(info) { +def releasecomponent(info, String dryrunpush) { def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -111,10 +111,10 @@ def releasecomponent(info) { getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) def String buildstamp = info.meta.buildstampinfo.buildstamp - gitrelease(uniquefolderforrepo, buildstamp) + gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) } -def releasehierarchy(buildmeta, String maxdepth) { +def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { int md = maxdepth.toInteger() resetfolders() preconditions() @@ -125,7 +125,7 @@ def releasehierarchy(buildmeta, String maxdepth) { // For each component for (key in componentReleaseInfo.keySet()) { def info = componentReleaseInfo[key] - releasecomponent(info) + releasecomponent(info, dryrunpush) } return [markunstable, unstablereason] } From e4f8dab2bbee21d2504f8466f2f60cd9eb00717b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:39:53 +0000 Subject: [PATCH 195/589] Move release functionality into Common_scripts --- release/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 264bd53..6db301c 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -31,7 +31,7 @@ def setunstable(String reason) { def summaryappend(String summary) { def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary + "\n") + writeFile(file: summaryfile, text: currentsummarycontents + summary) } def checkoutreleaserepo(info, String checkoutfolder) { @@ -112,6 +112,8 @@ def releasecomponent(info, String dryrunpush) { def String buildstamp = info.meta.buildstampinfo.buildstamp gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) + + summaryappend("\n") } def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { From 4e140fe10a98a0b28c82bfc39c86442dc5f22a64 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:07:08 +0000 Subject: [PATCH 196/589] Check for changes occurring --- release/do.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 6db301c..0d8e0de 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -73,11 +73,14 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { + def String precommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" // Add anything new sh("git add .") summaryappend("Add,") // Perform git commit (if necessary) - def changesoccurred = sh(script: "git status --porcelain", returnStdout: true) + def changes = sh(script: "git status --porcelain", returnStdout: true) + boolean changesoccurred = changes?.trim() if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") @@ -85,6 +88,8 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { else { summaryappend("No-commit,") } + def String postcommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${precommitshacode}" // Tag with buildstamp sh("git tag ${buildstamp}") summaryappend("Tag (${buildstamp}),") From 2a7e51f64d6c2915e65cebe714611f771637519c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:09:48 +0000 Subject: [PATCH 197/589] Check for changes occurring --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0d8e0de..cd49d41 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -73,7 +73,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { - def String precommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" // Add anything new sh("git add .") @@ -88,7 +88,7 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { else { summaryappend("No-commit,") } - def String postcommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${precommitshacode}" // Tag with buildstamp sh("git tag ${buildstamp}") From 0489ee2d82472b3b65410f91e7e199ff41e1b9f0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:28:15 +0000 Subject: [PATCH 198/589] Check for changes occurring --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index cd49d41..e72d6f2 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -81,6 +81,7 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { // Perform git commit (if necessary) def changes = sh(script: "git status --porcelain", returnStdout: true) boolean changesoccurred = changes?.trim() + echo "changesoccurred: ${changesoccurred}" if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") From ae1dcc41bf920341c60f6c12743b2e96102a928e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:32:17 +0000 Subject: [PATCH 199/589] Update --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index e72d6f2..11eb648 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -81,11 +81,11 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { // Perform git commit (if necessary) def changes = sh(script: "git status --porcelain", returnStdout: true) boolean changesoccurred = changes?.trim() - echo "changesoccurred: ${changesoccurred}" + echo "changes: ${changes}\nchangesoccurred: ${changesoccurred}" if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") - } + else { summaryappend("No-commit,") } From eb42ae659889748fac6e9150450cf6cbdd0e30b4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:34:57 +0000 Subject: [PATCH 200/589] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 11eb648..d7f05f7 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -85,7 +85,7 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") - + } else { summaryappend("No-commit,") } From 68cf41bc6f2f7994d4daaf261f584278317783cf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:06:40 +0000 Subject: [PATCH 201/589] Check for changes occurring --- meta/folder.groovy | 2 +- release/do.groovy | 167 +++++++++++++++++++++++++++++++++------------ 2 files changed, 125 insertions(+), 44 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9c5367e..cfce68f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -568,7 +568,7 @@ class MetaInfo extends GISJsonSerialised { } - String folder() { + static String folder() { return METAFOLDERNAME } diff --git a/release/do.groovy b/release/do.groovy index d7f05f7..80e10de 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -6,6 +6,8 @@ summaryfile = "${summaryfolder}/summary.txt" releaserepofolder = "${WORKSPACE}/releaserepos" tempfolder = "${WORKSPACE}/temp" +metascript = autobuilder.getmetafolderscript() + def resetfolders() { dir(summaryfolder) { deleteDir() @@ -34,21 +36,36 @@ def summaryappend(String summary) { writeFile(file: summaryfile, text: currentsummarycontents + summary) } -def checkoutreleaserepo(info, String checkoutfolder) { - def String releaserepo = info.meta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component - dir(releaserepofolder) { - sh("git clone -n --depth 1 ${releaserepo} ${checkoutfolder}") +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = new metascript.MetaInfo(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(releasemetafolderpath) } + return previousreleasemeta } -def getcomponentartifacts(info, String repofolder, String tempfolder) { +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = new metascript.MetaInfo(this) + dir(releaserepofolder) { + sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + // Clear all existing files + sh("git rm *") + } + return previousreleasemeta +} + +def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - for (artifactfilter in info.meta.releaseinfo.releaseitems) { + for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter - if (info.offsetpath) { - filterpath = info.offsetpath + '/' + artifactfilter + if (buildinfo.offsetpath) { + filterpath = buildinfo.offsetpath + '/' + artifactfilter } // Copy relevant artifacts to temp folder echo "Copying artifacts: ${filterpath} to ${tempfolder}" @@ -62,7 +79,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { target: tempfolder ) // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + info.offsetpath + def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { sh("mv * ${repofolder}") } @@ -70,59 +87,123 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { } } -def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean changed = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + return changed +} + +def ArrayList maketaglist(String buildstamp, String otherreleasetags) { + ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + taglist.add(othertagitem.trim()) + } + return taglist +} + +def boolean releasegittag(String postcommitshacode, String tag) { + // Tag + sh("git tag ${tag}") + summaryappend("(${tag}),") +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend("Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend("Push") + } +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") + def String buildstamp = buildinfo.meta.buildstampinfo.buildstamp dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - // Add anything new - sh("git add .") - summaryappend("Add,") - // Perform git commit (if necessary) - def changes = sh(script: "git status --porcelain", returnStdout: true) - boolean changesoccurred = changes?.trim() - echo "changes: ${changes}\nchangesoccurred: ${changesoccurred}" - if (changesoccurred) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") + + releasegitadd() + + def boolean changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamp) } else { + releasegitreset() summaryappend("No-commit,") } + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${precommitshacode}" - // Tag with buildstamp - sh("git tag ${buildstamp}") - summaryappend("Tag (${buildstamp}),") - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend("Dry-Run-Push,") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend("Push") + echo "Post commit sha code: ${postcommitshacode}" + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + + def boolean tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) } } } -def releasecomponent(info, String dryrunpush) { - def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" +def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { + def buildmeta = buildinfo.meta + def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - summaryappend("${info.meta.productinfo.productname} ${info.meta.productinfo.subcomponentname}: ") - checkoutreleaserepo(info, uniquefolderforrepo) + summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) - getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) - def String buildstamp = info.meta.buildstampinfo.buildstamp - gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, otherreleasetags, dryrunpush) summaryappend("\n") } -def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { +def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { int md = maxdepth.toInteger() resetfolders() preconditions() @@ -132,8 +213,8 @@ def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { // For each component for (key in componentReleaseInfo.keySet()) { - def info = componentReleaseInfo[key] - releasecomponent(info, dryrunpush) + def buildinfo = componentReleaseInfo[key] + releasecomponent(buildinfo, otherreleasetags, dryrunpush) } return [markunstable, unstablereason] } From 01b5194cabc2f4023c011b8f1f128b77c6358ee2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:16:10 +0000 Subject: [PATCH 202/589] Check for changes occurring --- meta/folder.groovy | 4 ++++ release/do.groovy | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cfce68f..13b1fe5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -624,6 +624,10 @@ class MetaInfo extends GISJsonSerialised { } return releasemetauniquecollection } + + MetaInfo getNewMetaObject(script) { + return new MetaInfo(script) + } } MetaInfo metareset(script) { diff --git a/release/do.groovy b/release/do.groovy index 80e10de..9d3e09a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -37,7 +37,7 @@ def summaryappend(String summary) { } def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = new metascript.MetaInfo(this) + def previousreleasemeta = buildmeta.getNewMetaObject(this) // See if there is meta data file from the current release def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" if (fileExists(releasemetafolderpath)) { @@ -49,7 +49,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) { def checkoutreleaserepo(buildmeta, String checkoutfolder) { def String releaserepo = buildmeta.releaseinfo.releaserepoURL // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - def previousreleasemeta = new metascript.MetaInfo(this) + def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) From d532ddbdb5c7a0966becac910025ec9a82f502bb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:21:49 +0000 Subject: [PATCH 203/589] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 9d3e09a..576a188 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -41,7 +41,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) { // See if there is meta data file from the current release def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(releasemetafolderpath) + previousreleasemeta = metascript.metafromfolder(checkoutfolder) } return previousreleasemeta } From 42d175974e6ea34722219476a44c9813378f4d0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:27:03 +0000 Subject: [PATCH 204/589] Check for changes occurring --- release/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 576a188..644b995 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -52,7 +52,9 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { // Clear all existing files sh("git rm *") } From 47545c5206c4a466f0eeac1357e85859040786b9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:29:48 +0000 Subject: [PATCH 205/589] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 644b995..6cf55e2 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -56,7 +56,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { // Clear all existing files - sh("git rm *") + sh("git rm -r *") } return previousreleasemeta } From 75855fd9fb152e51978f5b9f785dd3cc5920a04e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:32:39 +0000 Subject: [PATCH 206/589] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 6cf55e2..bcdc419 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -160,7 +160,7 @@ def releasegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildinfo.meta.buildstampinfo.buildstamp + def String buildstamp = buildmeta.buildstampinfo.buildstamp dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From 5926f63c7bdd534b5b4c9de784d15ad6dc9e6ee9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 16:15:58 +0000 Subject: [PATCH 207/589] Check for changes occurring --- release/do.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release/do.groovy b/release/do.groovy index bcdc419..a86bf04 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -129,6 +129,11 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { } def boolean releasegittag(String postcommitshacode, String tag) { + // Check if tag exists already + def retvalue = sh(script: "git rev-parse ${tag} >2 /dev/null", returnStdout: true, returnStatus: true) + echo "retvalue: ${retvalue} class: ${retvalue.getClass()}" + + // Tag sh("git tag ${tag}") summaryappend("(${tag}),") From 476c771144c259f23222061fcd8490615ab99ecb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 16:38:43 +0000 Subject: [PATCH 208/589] Check for changes occurring --- release/do.groovy | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a86bf04..2438b54 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -129,14 +129,26 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { } def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false // Check if tag exists already - def retvalue = sh(script: "git rev-parse ${tag} >2 /dev/null", returnStdout: true, returnStatus: true) - echo "retvalue: ${retvalue} class: ${retvalue.getClass()}" - - - // Tag - sh("git tag ${tag}") - summaryappend("(${tag}),") + def existingshacode = sh(script: "git rev-parse ${tag}", returnStdout: true).trim() + if (existingshacode?.trim()) { // If tag/sha code does exist + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!),") + } + else { + summaryappend("(${tag} - Already added),") + } + } + else { // If sha code does not exist already + // Tag + sh("git tag ${tag}") + summaryappend("(${tag}),") + tagadded = true + } + return tagadded } def boolean releasegittags(String postcommitshacode, ArrayList taglist) { From 6494069af5d92a59fe9c552c77a3938eb1756023 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 16:48:17 +0000 Subject: [PATCH 209/589] Check for changes occurring --- release/do.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2438b54..e69f09b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -125,6 +125,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } + echo "Taglist: ${taglist}" return taglist } @@ -136,16 +137,16 @@ def boolean releasegittag(String postcommitshacode, String tag) { if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!),") + summaryappend("(${tag} - Could not add!)") } else { - summaryappend("(${tag} - Already added),") + summaryappend("(${tag} - Already added)") } } else { // If sha code does not exist already // Tag sh("git tag ${tag}") - summaryappend("(${tag}),") + summaryappend("(${tag})") tagadded = true } return tagadded From 9c12e97ea42a82e7a92d6843d941a684815e04e3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:16:45 +0000 Subject: [PATCH 210/589] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index e69f09b..1fe7cd4 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -125,7 +125,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } - echo "Taglist: ${taglist}" + echo "Taglist: ${taglist} size: ${taglist.size()}" return taglist } From 192678568584e1e2e0052b2db99560c478d4e81d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:20:36 +0000 Subject: [PATCH 211/589] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1fe7cd4..788838b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -121,7 +121,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { ArrayList taglist = new ArrayList() taglist.add(buildstamp) - ArrayList othertaglist = otherreleasetags.split(',') + ArrayList othertaglist = otherreleasetags.trim().split(',') for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } From e1671aba544d1021fa2939b094219df6defe5e0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:36:43 +0000 Subject: [PATCH 212/589] Check for changes occurring --- release/do.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 788838b..5d4f2e9 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -119,9 +119,12 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { def ArrayList maketaglist(String buildstamp, String otherreleasetags) { ArrayList taglist = new ArrayList() - taglist.add(buildstamp) + taglist.add(buildstamp.trim()) + echo "Taglist: ${taglist} size: ${taglist.size()}" + ArrayList othertaglist = otherreleasetags.trim().split(',') + echo "othertaglist: ${othertaglist} size: ${othertaglist.size()}" for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } @@ -178,7 +181,7 @@ def releasegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp + def String buildstamp = buildmeta.buildstampinfo.buildstamp.trim() dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From 1500adb084374cf918b98b67b81272ae1c7172df Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:41:40 +0000 Subject: [PATCH 213/589] Check for changes occurring --- release/do.groovy | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 5d4f2e9..7b7c07c 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -118,15 +118,14 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { } def ArrayList maketaglist(String buildstamp, String otherreleasetags) { - ArrayList taglist = new ArrayList() - taglist.add(buildstamp.trim()) + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) - echo "Taglist: ${taglist} size: ${taglist.size()}" - - ArrayList othertaglist = otherreleasetags.trim().split(',') - echo "othertaglist: ${othertaglist} size: ${othertaglist.size()}" + def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { - taglist.add(othertagitem.trim()) + if (othertagitem?.trim()) { + taglist.add(othertagitem.trim()) + } } echo "Taglist: ${taglist} size: ${taglist.size()}" return taglist @@ -181,7 +180,7 @@ def releasegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp.trim() + def String buildstamp = buildmeta.buildstampinfo.buildstamp dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From b380eef774e2a3d8c79f64bbc7767fe00983c788 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 18:20:30 +0000 Subject: [PATCH 214/589] Restoration of subtree contents --- versionnumbering/generate.groovy | 39 ++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index a7b583a..925ed65 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -10,18 +10,43 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + def int changecount + if (isUnix()) { + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() + changecount = stdout.readLines()[1] as int + } echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" - - gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + + if (isUnix()) { + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git rev-parse HEAD").trim() + gitrevision = stdout.readLines()[1] + } echo "Revision: ${gitrevision}" - - remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + + if (isUnix()) { + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git remote show origin").trim() + remoteinfo = stdout.readLines()[1..-1] + } echo "Remote info: ${remoteinfo}" - - remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + + if (isUnix()) { + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git config --get remote.origin.url").trim() + remoteurl = stdout.readLines()[1] + } echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 10e2a571787bae9dedf1a5d2267d85df0dfd20e8 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Fri, 30 Nov 2018 18:26:07 +0000 Subject: [PATCH 215/589] Merge commit 'b380eef774e2a3d8c79f64bbc7767fe00983c788' into 1.10003.j20-comscr --- release/do.groovy | 135 ---------------------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 release/do.groovy diff --git a/release/do.groovy b/release/do.groovy deleted file mode 100644 index 6db301c..0000000 --- a/release/do.groovy +++ /dev/null @@ -1,135 +0,0 @@ -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -releaserepofolder = "${WORKSPACE}/releaserepos" -tempfolder = "${WORKSPACE}/temp" - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - writeFile(file: summaryfile, text: "Release summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason = reason -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) -} - -def checkoutreleaserepo(info, String checkoutfolder) { - def String releaserepo = info.meta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component - dir(releaserepofolder) { - sh("git clone -n --depth 1 ${releaserepo} ${checkoutfolder}") - } -} - -def getcomponentartifacts(info, String repofolder, String tempfolder) { - // Copy relevant artifacts to temp folder - summaryappend("Artifacts: ") - for (artifactfilter in info.meta.releaseinfo.releaseitems) { - def filterpath = artifactfilter - if (info.offsetpath) { - filterpath = info.offsetpath + '/' + artifactfilter - } - // Copy relevant artifacts to temp folder - echo "Copying artifacts: ${filterpath} to ${tempfolder}" - copyArtifacts( - filter: filterpath, - projectName: BUILD_JOB, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR - ], - target: tempfolder - ) - // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + info.offsetpath - dir(originalrootdir) { - sh("mv * ${repofolder}") - } - summaryappend("${artifactfilter},") - } -} - -def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { - summaryappend(" Git operations: ") - dir(releaserepo) { - // Add anything new - sh("git add .") - summaryappend("Add,") - // Perform git commit (if necessary) - def changesoccurred = sh(script: "git status --porcelain", returnStdout: true) - if (changesoccurred) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") - } - else { - summaryappend("No-commit,") - } - // Tag with buildstamp - sh("git tag ${buildstamp}") - summaryappend("Tag (${buildstamp}),") - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend("Dry-Run-Push,") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend("Push") - } - } -} - -def releasecomponent(info, String dryrunpush) { - def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - - summaryappend("${info.meta.productinfo.productname} ${info.meta.productinfo.subcomponentname}: ") - checkoutreleaserepo(info, uniquefolderforrepo) - - getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) - - def String buildstamp = info.meta.buildstampinfo.buildstamp - gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) - - summaryappend("\n") -} - -def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { - int md = maxdepth.toInteger() - resetfolders() - preconditions() - - // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = buildmeta.releasewalkerunique(md) - - // For each component - for (key in componentReleaseInfo.keySet()) { - def info = componentReleaseInfo[key] - releasecomponent(info, dryrunpush) - } - return [markunstable, unstablereason] -} - -return this; \ No newline at end of file From 2c15e3fb45bc603d41b1381d1f44e75cca1fc0dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 18:46:33 +0000 Subject: [PATCH 216/589] Check for changes occurring --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 7b7c07c..b2f09bd 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -127,7 +127,6 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { taglist.add(othertagitem.trim()) } } - echo "Taglist: ${taglist} size: ${taglist.size()}" return taglist } From 75708e8b1ded943eaa76016cf0d342341c68b9c0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 08:25:26 +0000 Subject: [PATCH 217/589] Make sure changecount is an integer --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 925ed65..f80d021 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -12,7 +12,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int changecount if (isUnix()) { - changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } else { def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() From 7b4ccb95da39981937b3fe4376e821ea46a30378 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 09:32:23 +0000 Subject: [PATCH 218/589] Add a summary on the console --- release/do.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index b2f09bd..2866793 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -5,6 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" releaserepofolder = "${WORKSPACE}/releaserepos" tempfolder = "${WORKSPACE}/temp" +summarystring = "" metascript = autobuilder.getmetafolderscript() @@ -23,6 +24,7 @@ def resetfolders() { def preconditions() { markunstable = false unstablereason = "" + summarystring = "" writeFile(file: summaryfile, text: "Release summary: \n") } @@ -34,6 +36,7 @@ def setunstable(String reason) { def summaryappend(String summary) { def String currentsummarycontents = readFile(file: summaryfile) writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary } def getpreviousreleasemeta(buildmeta, String checkoutfolder) { @@ -237,7 +240,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String def buildinfo = componentReleaseInfo[key] releasecomponent(buildinfo, otherreleasetags, dryrunpush) } - return [markunstable, unstablereason] + return [summarystring, markunstable, unstablereason] } return this; \ No newline at end of file From 77c0c47b5c09944bb6f7963e46d31207da96da99 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:19:27 +0000 Subject: [PATCH 219/589] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2866793..187fb01 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -3,9 +3,10 @@ unstablereason = "" summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" tempfolder = "${WORKSPACE}/temp" -summarystring = "" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" metascript = autobuilder.getmetafolderscript() @@ -136,8 +137,9 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def existingshacode = sh(script: "git rev-parse ${tag}", returnStdout: true).trim() - if (existingshacode?.trim()) { // If tag/sha code does exist + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true).trim() + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") @@ -147,9 +149,9 @@ def boolean releasegittag(String postcommitshacode, String tag) { summaryappend("(${tag} - Already added)") } } - else { // If sha code does not exist already + else { // If tag does not exist already // Tag - sh("git tag ${tag}") + sh("git tag \"${tag}\"") summaryappend("(${tag})") tagadded = true } @@ -243,4 +245,4 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String return [summarystring, markunstable, unstablereason] } -return this; \ No newline at end of file +return this; From 483c6ee6306aa1bfa7073d339a9b1b05abb1ffeb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:25:10 +0000 Subject: [PATCH 220/589] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 187fb01..a43beda 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -137,7 +137,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true).trim() + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD From 72072efecbb7796042d100c3a37068a6402f1395 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:43:26 +0000 Subject: [PATCH 221/589] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a43beda..dc834ea 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -137,7 +137,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true) + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\" 2> /dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD @@ -173,12 +173,12 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) def releasegitpush(String dryrunpush ) { if (dryrunpush == "true") { sh("git push -n --tags origin master") - summaryappend("Dry-Run-Push") + summaryappend(",Dry-Run-Push") setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") } else { sh("git push --tags origin master") - summaryappend("Push") + summaryappend(",Push") } } From 467ffd7ac0b4385506665742d5a607125d7916d7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:47:30 +0000 Subject: [PATCH 222/589] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index dc834ea..f71c3df 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -137,7 +137,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\" 2> /dev/null", returnStatus: true) + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD From df484224e7911770599998344c7a9416a342792e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 12:27:00 +0000 Subject: [PATCH 223/589] Tag associated source repos --- release/do.groovy | 86 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index f71c3df..a320775 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -5,6 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" @@ -65,6 +66,14 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { return previousreleasemeta } +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + dir(sourcerepofolder) { + sh("git clone --depth 1 ${sourcerepo} ${checkoutfolder}") + } +} + def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -112,13 +121,13 @@ def releasegitcommit(String buildstamp) { } // Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { +def checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision def boolean changed = (buildsha != previoussha) echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return changed + return [changed, buildsha] } def ArrayList maketaglist(String buildstamp, String otherreleasetags) { @@ -170,6 +179,42 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) return tagadded } +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(buildsha, taglist) { + boolean tagadded = false + + summaryappend("Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + def releasegitpush(String dryrunpush ) { if (dryrunpush == "true") { sh("git push -n --tags origin master") @@ -182,16 +227,34 @@ def releasegitpush(String dryrunpush ) { } } -def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Push") + } +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") def String buildstamp = buildmeta.buildstampinfo.buildstamp + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + + def boolean changed = false + def String buildsha = "" + def boolean tagadded = false dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" releasegitadd() - def boolean changed = checkforchange(buildmeta, previousreleasemeta) + (changed, buildsha)= checkforchange(buildmeta, previousreleasemeta) if (changed) { releasegitcommit(buildstamp) } @@ -203,14 +266,18 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherr def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - - def boolean tagadded = releasegittags(postcommitshacode, taglist) + tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { releasegitpush(dryrunpush) } } + dir(sourcerepo) { + tagadded = sourcegittags(buildsha, taglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } } def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { @@ -218,13 +285,16 @@ def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) + checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) + getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, otherreleasetags, dryrunpush) + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) summaryappend("\n") } From 995062473d2f0c27ca143832626f1cb08f796fb3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 13:25:00 +0000 Subject: [PATCH 224/589] Tag associated source repos --- release/do.groovy | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a320775..6992f81 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -121,13 +121,13 @@ def releasegitcommit(String buildstamp) { } // Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def checkforchange(buildmeta, previousreleasemeta) { +def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision def boolean changed = (buildsha != previoussha) echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return [changed, buildsha] + return changed } def ArrayList maketaglist(String buildstamp, String otherreleasetags) { @@ -196,17 +196,17 @@ def boolean sourcegittag(String shacode, String tag) { } else { // If tag does not exist already // Tag - sh("git tag \"${tag}\"") + sh("git tag \"${tag}\" ${shacode}") summaryappend("(${tag})") tagadded = true } return tagadded } -def sourcegittags(buildsha, taglist) { +def sourcegittags(String buildsha, List taglist) { boolean tagadded = false - summaryappend("Src Tag ") + summaryappend(",Src Tag ") for (tag in taglist) { if (sourcegittag(buildsha, tag)) { tagadded = true @@ -243,18 +243,14 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source summaryappend(" Git operations: ") def String buildstamp = buildmeta.buildstampinfo.buildstamp - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - - def boolean changed = false def String buildsha = "" - def boolean tagadded = false dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" releasegitadd() - (changed, buildsha)= checkforchange(buildmeta, previousreleasemeta) + def boolean changed = checkforchange(buildmeta, previousreleasemeta) if (changed) { releasegitcommit(buildstamp) } @@ -266,14 +262,19 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - tagadded = releasegittags(postcommitshacode, taglist) + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { releasegitpush(dryrunpush) } } dir(sourcerepo) { - tagadded = sourcegittags(buildsha, taglist) + def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.buildstampinfo.buildstamp}" + def String buildsha = buildmeta.gitinfo.gitrevision + + def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) if (tagadded) { sourcegitpush(dryrunpush) } @@ -306,7 +307,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) - + // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] From 8a08eefc068043ff5e775162760f94709bf1a21f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 13:27:12 +0000 Subject: [PATCH 225/589] Tag associated source repos --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 6992f81..ce2aef0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -243,7 +243,6 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source summaryappend(" Git operations: ") def String buildstamp = buildmeta.buildstampinfo.buildstamp - def String buildsha = "" dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From 70473b38d8f4a5a5d01a010be067363778d18ce9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 14:19:59 +0000 Subject: [PATCH 226/589] Tag associated source repos --- release/do.groovy | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index ce2aef0..4a00491 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -69,8 +69,20 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - dir(sourcerepofolder) { - sh("git clone --depth 1 ${sourcerepo} ${checkoutfolder}") + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard" + git clean -fdx + git pull + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } } } @@ -269,7 +281,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.buildstampinfo.buildstamp}" + def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" def String buildsha = buildmeta.gitinfo.gitrevision def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) From f08e4ba3aab0c53ed2740f8b517e54c7316e0eae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 14:22:22 +0000 Subject: [PATCH 227/589] Tag associated source repos --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 4a00491..f0c6c8b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -72,7 +72,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { if (fileExists(checkoutfolder)) { dir(checkoutfolder) { sh(""" - git reset --hard" + git reset --hard git clean -fdx git pull """ From e02d609283aa6f14a0b1498f18b6cc45ca021a14 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 14:52:01 +0000 Subject: [PATCH 228/589] Make sure product name and sub component names can be used in tags --- meta/folder.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 13b1fe5..7ab5744 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -31,6 +31,15 @@ class ProductInfo extends GISJsonSerialised { } void set(String productname, String subcomponentname, String jirakey) { + // Make sure the fields can be used in git tags + def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\"", , returnStatus: true) + if (pstatus > 0) { + script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") + } + def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\"", , returnStatus: true) + if (sstatus > 0) { + script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") + } this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jirakey From 9044b468b62882c353351d94b5c604629a8dd3fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 15:00:02 +0000 Subject: [PATCH 229/589] Make sure product name and sub component names can be used in tags --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7ab5744..e1e38aa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -32,11 +32,11 @@ class ProductInfo extends GISJsonSerialised { void set(String productname, String subcomponentname, String jirakey) { // Make sure the fields can be used in git tags - def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\"", , returnStatus: true) + def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) if (pstatus > 0) { script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") } - def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\"", , returnStatus: true) + def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) if (sstatus > 0) { script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") } From 391001aa3bd2a805da83cdc29f951873c9cc0b76 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 23:27:47 +0000 Subject: [PATCH 230/589] Make sure all excess local tags are removed from the source repo at the start --- release/do.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/do.groovy b/release/do.groovy index f0c6c8b..ecdd4eb 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -75,6 +75,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { git reset --hard git clean -fdx git pull + # Remove all tags that are not already on the remote + git fetch --tags --prune """ ) } From e419aac74f0ed95134730adb0154e6f7f1830052 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 23:53:17 +0000 Subject: [PATCH 231/589] Make sure all excess local tags are removed from the source repo at the start --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index ecdd4eb..c8e1ce6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -76,7 +76,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { git clean -fdx git pull # Remove all tags that are not already on the remote - git fetch --tags --prune + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null """ ) } From 9abb04a84816cde72ddde80e15ac4b4b78b423bc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 2 Dec 2018 17:04:33 +0000 Subject: [PATCH 232/589] Distinguish src and release repo pushes --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index c8e1ce6..169c292 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -245,12 +245,12 @@ def releasegitpush(String dryrunpush ) { def sourcegitpush(String dryrunpush ) { if (dryrunpush == "true") { sh("git push -n --tags") - summaryappend(",Dry-Run-Push") + summaryappend(",Src-Dry-Run-Push") setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") } else { sh("git push --tags") - summaryappend(",Push") + summaryappend(",Src Push") } } From 9cb07c5f1617d0b59d47d0ac486f1d8650827bf3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 2 Dec 2018 18:08:43 +0000 Subject: [PATCH 233/589] Make sure product name and sub component names can be used in tags --- meta/folder.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index e1e38aa..1edd45c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -434,6 +434,7 @@ class MetaInfo extends GISJsonSerialised { ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo + private LinkedHashMap custominfo private ArrayList associatedmeta MetaInfo(script) { @@ -446,6 +447,7 @@ class MetaInfo extends GISJsonSerialised { checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() publishableinfo = new ArrayList() + custominfo = new LinkedHashMap() associatedmeta = new ArrayList() } @@ -467,6 +469,14 @@ class MetaInfo extends GISJsonSerialised { configurationinfo.add(ci) } + void addCustomInfo(String customId, String customValue) { + custominfo.put(customId.trim(), customValue) + } + + String getCustomInfo(String customId) { + return custominfo.getAt(customId.trim()) + } + List publishableinfoserialised() { def publishable = [] for (publishableitem in publishableinfo) { @@ -504,6 +514,7 @@ class MetaInfo extends GISJsonSerialised { pb.deserialiseJson(rawpublishable) publishableinfo.add(pb) } + custominfo = rawmeta['custominfo'] for (rawrelatedmeta in rawmeta['associatedmeta']) { def rm = new RelatedMetaInfo(script) rm.deserialiseJson(rawrelatedmeta) @@ -571,6 +582,7 @@ class MetaInfo extends GISJsonSerialised { 'configuration' : configurationinfoserialised(), 'checksuminfo' : checksuminfo.serialisationdata(), 'publishableinfo' : publishableinfoserialised(), + 'custominfo' : custominfo, 'associatedmeta' : associatedmetaserialised(), ] return data From bf616a38a1e9ef837be0ca6e94d9b9cfb9c4bb56 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 08:37:23 +0000 Subject: [PATCH 234/589] Add bat variant of git commands --- meta/folder.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1edd45c..643e203 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -32,11 +32,23 @@ class ProductInfo extends GISJsonSerialised { void set(String productname, String subcomponentname, String jirakey) { // Make sure the fields can be used in git tags - def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) + def int pstatus = 0 + if (script.isUnix()) { + pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + pstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${productname}\" > NUL 2> NUL", , returnStatus: true) + } if (pstatus > 0) { script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") } - def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) + def int sstatus = 0 + if (script.isUnix()) { + sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + sstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${subcomponentname}\" > NUL 2> NUL", , returnStatus: true) + } if (sstatus > 0) { script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") } From 66082496ba822a5d2883823f745b97cc4e54a89b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 08:48:11 +0000 Subject: [PATCH 235/589] Add bat variant of git commands --- versionnumbering/generate.groovy | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f80d021..ae7e5f4 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -10,42 +10,35 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - def int changecount + def int changecount = 0 if (isUnix()) { changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } else { - def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() - changecount = stdout.readLines()[1] as int - } + changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() + } echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" - if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() } else { - def stdout = bat(returnStdout:true , script: "git rev-parse HEAD").trim() - gitrevision = stdout.readLines()[1] + gitrevision = bat(returnStdout: true, script: "@git rev-parse HEAD").trim() } echo "Revision: ${gitrevision}" - if (isUnix()) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() } else { - def stdout = bat(returnStdout:true , script: "git remote show origin").trim() - remoteinfo = stdout.readLines()[1..-1] + remoteinfo = bat(returnStdout: true, script: "@git remote show origin").trim() } echo "Remote info: ${remoteinfo}" - if (isUnix()) { remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() } else { - def stdout = bat(returnStdout:true , script: "git config --get remote.origin.url").trim() - remoteurl = stdout.readLines()[1] + remoteurl = bat(returnStdout: true, script: "@git config --get remote.origin.url").trim() } echo "Remote URL: ${remoteurl}" } From 95caf56875baa6ed8b58fd36c7e9cf9855a3ea81 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 13:28:28 +0000 Subject: [PATCH 236/589] Make installer build use meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 643e203..15f77d0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -605,8 +605,8 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), - ArrayList releasedfromsubmeta = new ArrayList(), + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { def String artifactlist = METAARTIFACTS From a5cd7e43acf791622620f270e12decfe2953ff3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 14:47:13 +0000 Subject: [PATCH 237/589] Import specific job binaries; Add imported meta to this jobs meta --- import/importer.groovy | 6 ++++-- install/installhelper.groovy | 19 +++++-------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..bf040ce 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,7 +1,8 @@ -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, + excludes: excludes, fingerprintArtifacts: fingerprint, projectName: jenkinsjob.trim(), selector: specific(buildnumber.toString()), @@ -10,9 +11,10 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St return meta.importrelated(destpath) } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, + excludes: excludes, fingerprintArtifacts: fingerprint, projectName: jenkinsjob.trim(), selector: [ diff --git a/install/installhelper.groovy b/install/installhelper.groovy index f523799..fdd6016 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,30 +1,21 @@ // Given details about artifacts to import and details of installers to build, import artifacts that are required -def artifactsImporter(artifactDetails, installerDetails) { +def artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] - //echo "To build: ${installer} use artifactKey: ${artifactKey}" - (sourceProject, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { - //echo "Artifact info: ${artifactInfo}" - copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" - def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" - def folderToCopy = "${destFolder}\\version" - echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" - fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) - //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) - def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - artifactVersion = gh.getSHAFromVersion("${destFolder}/version") - artifactFetched.put(artifactKey, artifactVersion) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destpath, toCopy, notToCopy, true) } } return artifactFetched } // Export any required binaries to the /bin folder -def exportBinaries(artifactDetails, installerDetails) { +def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { From 5aae28adceb6c3d13281be015d9f9519b3e34262 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 15:06:26 +0000 Subject: [PATCH 238/589] Import specific job binaries; Add imported meta to this jobs meta --- install/installhelper.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index fdd6016..206a6c1 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -6,9 +6,8 @@ def artifactsImporter(meta, artifactDetails, installerDetails) { for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) - env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destpath, toCopy, notToCopy, true) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) } } return artifactFetched From 2d679105c8c4f23cf7e46940d8547c36d9e99351 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 15:36:06 +0000 Subject: [PATCH 239/589] Import specific job binaries; Add imported meta to this jobs meta --- install/installhelper.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 206a6c1..0155c7f 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,5 +1,5 @@ // Given details about artifacts to import and details of installers to build, import artifacts that are required -def artifactsImporter(meta, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") @@ -8,6 +8,7 @@ def artifactsImporter(meta, artifactDetails, installerDetails) { (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) + artifactFetched.put(artifactKey, importmeta) } } return artifactFetched From db6163fff766bbaecc039aa3ccafa9df7b2b0aee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Dec 2018 18:30:15 +0000 Subject: [PATCH 240/589] Add publishing script --- publish/do.groovy | 313 ++++++++++++++++++++++++++++++++++++++ versioncontrol/git.groovy | 18 ++- 2 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 publish/do.groovy diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..2c5a018 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,313 @@ +markunstable = false +unstablereason = "" + +FTP_CLIENT_LOCATION='ftp' +FTP_URL='ie1.hostedftp.com' +FTP_USER='ftp@globalinkjetsystems.com' +FTP_CUSTOMER_ROOT='/Customers/' +FTP_PASSWORD='ramshorn42' +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } +} + +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.meta + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" + def publishableitems = releasemeta.publishableinfo + def ArrayList publishablePaths = new ArrayList() + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean changed = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + return changed +} + +def ArrayList maketaglist(String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + taglist.add(othertagitem.trim()) + } + } + return taglist +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def publish(releasemeta, String uniquefolderforrepo) { + summaryappend(" Git operations: ") + def String buildstamp = releasemeta.buildstampinfo.buildstamp + + dir(uniquefolderforrepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + releasegitadd() + + def boolean changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamp) + } + else { + releasegitreset() + summaryappend("No-commit,") + } + + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } +} + +def recallcomponent(releaseinfo) { + def releasemeta = releaseinfo.meta + def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + + summaryappend("\n") +} + +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = releasemeta.releasewalkerunique(md) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + recallcomponent(releaseinfo) + } + // publish(releasemeta, uniquefolderforrepo) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index cb9354b..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,3 +1,7 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { @@ -6,7 +10,16 @@ String getGitInfo(String workingSetRootPath) { } } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +@NonCPS +def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return new SparseCheckoutPaths(sparsecheckoutpaths) +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRoot != "") { reporootpath = "${reporootenv.trim()}/${repoRoot}" @@ -28,7 +41,8 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + sparsecheckoutconfig(sparseitemlist) ], submoduleCfg: [], ]) From 9ea146b13318e6f5beedd7dd917d6cda79ac60b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 11:22:13 +0000 Subject: [PATCH 241/589] Only checkout repositories that have publishable items --- publish/do.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 2c5a018..c7084d0 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,11 +104,13 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" def publishableitems = releasemeta.publishableinfo def ArrayList publishablePaths = new ArrayList() - for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } def releasegitadd() { From a7f6f97d7720cebc81be274449a9634d3569cfbc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:15:10 +0000 Subject: [PATCH 242/589] Build up folder structure for FTP --- publish/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c7084d0..4d02b4f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -281,9 +281,9 @@ def publish(releasemeta, String uniquefolderforrepo) { } } -def recallcomponent(releaseinfo) { +def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -305,7 +305,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo) + recallcomponent(releaseinfo, customer, softwarerelease) } // publish(releasemeta, uniquefolderforrepo) From 67ca5024c206ccfd03e322717df27ae1458332f9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:59:24 +0000 Subject: [PATCH 243/589] Change root path for transfer; Add an example FTP operation --- publish/do.groovy | 53 ++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 4d02b4f..b953f4c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,11 +1,6 @@ markunstable = false unstablereason = "" -FTP_CLIENT_LOCATION='ftp' -FTP_URL='ie1.hostedftp.com' -FTP_USER='ftp@globalinkjetsystems.com' -FTP_CUSTOMER_ROOT='/Customers/' -FTP_PASSWORD='ramshorn42' summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" @@ -250,40 +245,32 @@ def sourcegitpush(String dryrunpush ) { } } -def publish(releasemeta, String uniquefolderforrepo) { - summaryappend(" Git operations: ") - def String buildstamp = releasemeta.buildstampinfo.buildstamp +def String FTP_CLIENT_LOCATION='ftp' +def String FTP_URL='ie1.hostedftp.com' +def String FTP_USER='ftp@globalinkjetsystems.com' +def String FTP_CUSTOMER_ROOT='/Customers/' +def String FTP_PASSWORD='ramshorn42' - dir(uniquefolderforrepo) { - def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Pre commit sha code: ${precommitshacode}" - - releasegitadd() +def publish() { + summaryappend(" FTP operations: ") - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamp) - } - else { - releasegitreset() - summaryappend("No-commit,") - } - - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" - - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - def tagadded = releasegittags(postcommitshacode, taglist) - - if (changed || tagadded) { - releasegitpush(dryrunpush) - } + dir(releaserepofolder) { + sh """#!/bin/bash + ftp -n -v ${FTP_URL} << EOT + ascii + user ${FTP_USER} ${FTP_PASSWORD} + prompt + cd ${FTP_CUSTOMER_ROOT} + ls -la + bye + EOT + """ } } def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -307,7 +294,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, customer, softwarerelease) } - // publish(releasemeta, uniquefolderforrepo) + publish() return [summarystring, markunstable, unstablereason] } From b178b86f6ee9199bdc418f5cf79d28b6505b7a00 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 14:26:20 +0000 Subject: [PATCH 244/589] Build up folder structure for FTP --- publish/do.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b953f4c..46b3fdd 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -245,15 +245,15 @@ def sourcegitpush(String dryrunpush ) { } } -def String FTP_CLIENT_LOCATION='ftp' -def String FTP_URL='ie1.hostedftp.com' -def String FTP_USER='ftp@globalinkjetsystems.com' -def String FTP_CUSTOMER_ROOT='/Customers/' -def String FTP_PASSWORD='ramshorn42' def publish() { summaryappend(" FTP operations: ") + def String FTP_CLIENT_LOCATION='ftp' + def String FTP_URL='ie1.hostedftp.com' + def String FTP_USER='ftp@globalinkjetsystems.com' + def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { sh """#!/bin/bash ftp -n -v ${FTP_URL} << EOT From 4230e4dcd33918892e2f4ac90fafb688dad4ba87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:14:29 +0000 Subject: [PATCH 245/589] Use ncftpput to transfer --- publish/do.groovy | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 46b3fdd..13a280a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -106,6 +106,9 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } + else { + summaryappend("None ") + } } def releasegitadd() { @@ -245,32 +248,26 @@ def sourcegitpush(String dryrunpush ) { } } - -def publish() { +def publish(String customer) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' def String FTP_URL='ie1.hostedftp.com' def String FTP_USER='ftp@globalinkjetsystems.com' - def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { + summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") sh """#!/bin/bash - ftp -n -v ${FTP_URL} << EOT - ascii - user ${FTP_USER} ${FTP_PASSWORD} - prompt - cd ${FTP_CUSTOMER_ROOT} - ls -la - bye - EOT + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ + summaryappend(", Transferred ") } } -def recallcomponent(releaseinfo, String customer, String softwarerelease) { +def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -292,9 +289,9 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo, customer, softwarerelease) + recallcomponent(releaseinfo, softwarerelease) } - publish() + publish(customer) return [summarystring, markunstable, unstablereason] } From e06d3705497981903880f07cb750004a800e5669 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:59:04 +0000 Subject: [PATCH 246/589] Use ncftpput to transfer --- publish/do.groovy | 200 +++------------------------------------------- 1 file changed, 10 insertions(+), 190 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 13a280a..8cdf9a3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -5,12 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -42,54 +37,7 @@ def summaryappend(String summary) { summarystring += summary } -def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = buildmeta.getNewMetaObject(this) - // See if there is meta data file from the current release - def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" - if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(checkoutfolder) - } - return previousreleasemeta -} - -def checkoutreleaserepo(buildmeta, String checkoutfolder) { - def String releaserepo = buildmeta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - def previousreleasemeta = buildmeta.getNewMetaObject(this) - dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") - } - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) - dir(checkoutfolder) { - // Clear all existing files - sh("git rm -r *") - } - return previousreleasemeta -} - -def checkoutsourcerepo(buildmeta, String checkoutfolder) { - def String sourcerepo = buildmeta.gitinfo.remoteurl - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } -} - +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -111,143 +59,6 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } } -def releasegitadd() { - // Add anything new - sh("git add .") - summaryappend("Add,") -} - -def releasegitreset() { - // Add anything new - sh("git reset --hard") - sh("git clean -fdx") - summaryappend("Reset,") -} - -def releasegitcommit(String buildstamp) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") -} - -// Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { - String buildsha = buildmeta.gitinfo.gitrevision - String previoussha = previousreleasemeta.gitinfo.gitrevision - - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return changed -} - -def ArrayList maketaglist(String buildstamp, String otherreleasetags) { - def ArrayList taglist = new ArrayList() - taglist.add(buildstamp) - - def ArrayList othertaglist = otherreleasetags.split(',') - for (othertagitem in othertaglist) { - if (othertagitem?.trim()) { - taglist.add(othertagitem.trim()) - } - } - return taglist -} - -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\"") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def boolean releasegittags(String postcommitshacode, ArrayList taglist) { - boolean tagadded = false - - summaryappend("Rel Tag ") - for (tag in taglist) { - if (releasegittag(postcommitshacode, tag)) { - tagadded = true - } - } - return tagadded -} - -def boolean sourcegittag(String shacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != shacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\" ${shacode}") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def sourcegittags(String buildsha, List taglist) { - boolean tagadded = false - - summaryappend(",Src Tag ") - for (tag in taglist) { - if (sourcegittag(buildsha, tag)) { - tagadded = true - } - } - return tagadded -} - -def releasegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend(",Dry-Run-Push") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend(",Push") - } -} - -def sourcegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags") - summaryappend(",Src-Dry-Run-Push") - setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags") - summaryappend(",Src Push") - } -} - def publish(String customer) { summaryappend(" FTP operations: ") @@ -278,6 +89,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { summaryappend("\n") } +def stripgithistory() { + dir(releaserepofolder) { + sh """#!/bin/bash + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + """ + } +} + def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() @@ -291,6 +110,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, softwarerelease) } + stripgithistory() publish(customer) return [summarystring, markunstable, unstablereason] From 8ba5a2c59a7407c8d993bd343793818953edbc7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 16:01:13 +0000 Subject: [PATCH 247/589] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 8cdf9a3..34d0e42 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -68,7 +68,7 @@ def publish(String customer) { def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { - summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") + summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ From 08e458d9d633bfca42d0dd39bb5e549066a6b277 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:12:45 +0000 Subject: [PATCH 248/589] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 34d0e42..a30bfc9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -92,7 +92,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf """ } } From 64a6b177de8bce34c2a37982ec5305bd62be2d96 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:53:08 +0000 Subject: [PATCH 249/589] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- artifacts/metaexport.groovy | 8 ++------ import/importer.groovy | 10 ++++++++-- versionnumbering/generate.groovy | 5 +++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b4d4a6d..0ca5da7 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,7 +4,6 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() - int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,14 +13,11 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - revisiontotal += importedmeta.buildstampinfo.revision + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders.add("${destFolder}/**") } } - def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) - return [artifactFolders, revisiontotal] + return artifactFolders } return this; diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..081226a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,6 @@ +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -7,7 +9,9 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +25,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ae7e5f4..e73782f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,4 +136,9 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + return newrevision +} + return this; \ No newline at end of file From e3bf9774ede490241fd0840e834e5230e4c9dc4d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 21:25:38 +0000 Subject: [PATCH 250/589] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- versionnumbering/generate.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e73782f..7e5568a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,6 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" return newrevision } From 2dbfbdab472fc59f27e233fb3f9fdf5759a99278 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:14:13 +0000 Subject: [PATCH 251/589] Remove .git files/folders before publishing --- publish/do.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..91f3b3e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepo}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs rm -rvf + find . -name ".gitignore" | xargs rm -rvf + find . -name ".gitmodules" | xargs rm -rvf + find . -iname ".gitattributes" | xargs rm -rvf """ } } From 6ac6396c6f7ffeac25f73b7c8344d561b81e859d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:29:40 +0000 Subject: [PATCH 252/589] Remove .git files/folders before publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 91f3b3e..ec9b53f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,7 +90,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { - echo "Stripping git history out of ${releaserepo}" + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash pwd From 7a4c46d342ac3671f42d2077be48d0bbf21176ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 14:01:09 +0000 Subject: [PATCH 253/589] Releasing to git was relying on the source code sha changing only; However it needs to take into account that a dependent component revision number changed too --- release/do.groovy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } From ca914eebd209abcefaca8ffbbd1f21847e9de156 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:20:25 +0000 Subject: [PATCH 254/589] Remove .git detritus from published folders --- publish/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index ec9b53f..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -94,10 +94,10 @@ def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash pwd - find . -type d -name ".git" | xargs rm -rvf - find . -name ".gitignore" | xargs rm -rvf - find . -name ".gitmodules" | xargs rm -rvf - find . -iname ".gitattributes" | xargs rm -rvf + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } From 5a1a55aa35a0b9d2a4fae58c9c3143947aa65e68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:22:15 +0000 Subject: [PATCH 255/589] Move product agnostic scripts into common subtree area --- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From fa3cc37a4d9ec6c79c55aaa09e2772aea9065977 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 16:47:07 +0000 Subject: [PATCH 256/589] Post subtree pull corrections --- publish/do.groovy | 7 +++- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/do.groovy | 14 ++++++- release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From e00d396c2b5dab238d2b0bff57abf0a1d0a0c082 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 17:49:02 +0000 Subject: [PATCH 257/589] Remove redundant common_scripts --- file/utils.groovy | 16 --- versionnumbering/versioninfo.py | 180 -------------------------------- 2 files changed, 196 deletions(-) delete mode 100644 file/utils.groovy delete mode 100644 versionnumbering/versioninfo.py diff --git a/file/utils.groovy b/file/utils.groovy deleted file mode 100644 index 0ffa47c..0000000 --- a/file/utils.groovy +++ /dev/null @@ -1,16 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - -// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents -def currentContents(file, newline = "${nl}", defaultContents = "") { - def contents = "" - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - -return this; diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py deleted file mode 100644 index 3d2a80d..0000000 --- a/versionnumbering/versioninfo.py +++ /dev/null @@ -1,180 +0,0 @@ -# encoding: utf-8 -''' -versionnumbering -- shortdesc - -versionnumbering generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess -import re -import datetime - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - thirdpartyversionprefix = "1.25000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().find("master") != -1: # Special case - return mastertypeversionprefix - - splits2 = re.split(r'(\d+)', sourcebranch) - splits = [] - hasdot = False - for split in splits2: - if split.isdigit(): - splits.append(split) - if split == ".": - hasdot = True - - if not hasdot: - print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) - return thirdpartyversionprefix - - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - sourcebranch = args.sourcebranch - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - - print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber) - - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) From 1eba3bf0162fead6284b53e63647c2f2456326d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 18:00:34 +0000 Subject: [PATCH 258/589] Remove compile retries --- compile/eclipsebuild/eclipseds5build.groovy | 52 ++++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..cf382b1 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,39 +142,35 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ } } - else { - echo "Not building ${build_variant} variant of project: ${project}" + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } - } catch (Exception e) { - echo "Build failed - Waiting for a while" - sleep(time: 4, unit: 'SECONDS') - error "Build failed..." } + else { + echo "Not building ${build_variant} variant of project: ${project}" + } + } catch (Exception e) { + error "Build failed..." } } } From 8ae90886986d45e2261d7a1674737241b85298a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:38:54 +0000 Subject: [PATCH 259/589] Fixing version stamping in windows --- versionnumbering/do.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 9b4a42e..0c53ce9 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -1,17 +1,20 @@ def stamp(convertdetails) { for (convertdetail in convertdetails) { - filetoconvert = convertdetail[0] - pattern = convertdetail[1] - replacement = convertdetail[2] + def filetoconvert = convertdetail[0] + def pattern = convertdetail[1] + def replacement = convertdetail[2] + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" + echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ - "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } else { bat """ - "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + "%${PYTHON}%" -u "%${stampscript}%" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" """ } } From fc461ba62b9caf0e2f7d1c8215f7aa309ee45a4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:46:44 +0000 Subject: [PATCH 260/589] Fixing version stamping in windows --- versionnumbering/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 0c53ce9..9f45f70 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -14,7 +14,7 @@ def stamp(convertdetails) { } else { bat """ - "%${PYTHON}%" -u "%${stampscript}%" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } } From 7bd8655a9224716056a3ec25beed4d4ba97ad6b3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:59:26 +0000 Subject: [PATCH 261/589] Fixing version stamping in windows --- versionnumbering/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 9f45f70..666aee3 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -4,9 +4,9 @@ def stamp(convertdetails) { def filetoconvert = convertdetail[0] def pattern = convertdetail[1] def replacement = convertdetail[2] - echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + //echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - echo "Using python: ${PYTHON} and script: ${stampscript}" + //echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" From 93ef7dfcf78b80b3eb033320caa9494a0052cb33 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 12:05:47 +0000 Subject: [PATCH 262/589] Revert "Remove compile retries" This reverts commit 90ee2b5c1c5c2bcf13af8d3c501114f41b3d8181. --- compile/eclipsebuild/eclipseds5build.groovy | 52 +++++++++++---------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index cf382b1..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,35 +142,39 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } + } catch (Exception e) { + echo "Build failed - Waiting for a while" + sleep(time: 4, unit: 'SECONDS') + error "Build failed..." } - else { - echo "Not building ${build_variant} variant of project: ${project}" - } - } catch (Exception e) { - error "Build failed..." } } } From 2371eff2e841e448c32bf7c2e84f5023a0b44e19 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 17:24:46 +0000 Subject: [PATCH 263/589] Move eclipse nios operation into common area --- compile/eclipsebuild/eclipseniosbuild.groovy | 103 +++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 compile/eclipsebuild/eclipseniosbuild.groovy diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy new file mode 100644 index 0000000..18c31d1 --- /dev/null +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -0,0 +1,103 @@ +// Gets the debug and optimization flags. +// Arg1 is "app" or "lib". Returns flags. +// Restore the debug and optimization levels, in case this is a dev machine. +// Arg1 is "app" or "lib". Arg2-3 is flags from getFlags. Returns nothing. +def setFlagsToRelease(NIOSEDS, project, wdir) { + if ( project == "app" ) { + if (isUnix()) { + sh """#!/bin/bash + set -x + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-app-update-makefile --app-dir=. --set-optimization=-O2 --set-debug-level= + """ + } + } + else if ( project == "lib" ) { + if (isUnix()) { + sh """#!/bin/bash + set -x + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-lib-update-makefile --lib-dir=. --set-optimization=-O2 --set-debug-level= + """ + } + } +} + +def remake_and_check(NIOSEDS, wdir) { + if (isUnix()) { + sh """#!/bin/bash + set -x + cd "${wdir}" + + # Use the makefile to clean and build + "${NIOSEDS}/nios2_command_shell.sh" make clean + if [ \$? -ne 0 ]; then exit 1; fi + + # Set the debug and optimization levels to production values + "${NIOSEDS}/nios2_command_shell.sh" make -j12 all + if [ \$? -ne 0 ]; then exit 1; fi + """ + } +} + +def generate_bsp(NIOSEDS, bspfolder) { + if (isUnix()) { + sh """#!/bin/bash + set -x + + "${NIOSEDS}/nios2_command_shell.sh" nios2-bsp-generate-files --bsp-dir "${bspfolder}" --log "${bspfolder}/bsp.log" --settings "${bspfolder}/settings.bsp" + """ + } +} + +def modifylib(NIOSEDS, bspfolder, buildconfig, wdir) { + if (isUnix()) { + sh """#!/bin/bash + set -x + + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-lib-update-makefile --lib-dir=. --set-bsp-dir "${bspfolder}" --set-active-build-config "${buildconfig}" + """ + } +} + +def modifyapp(NIOSEDS, bspfolder, wdir) { + if (isUnix()) { + sh """#!/bin/bash + set -x + + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-app-update-makefile --app-dir=. --set-bsp-dir "${bspfolder}" + """ + } +} + +def dump_elf_info(NIOSEDS, elffile, outputdir) { + sh """#!/bin/bash + set -x + + mkdir -p "${outputdir}" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name > "${outputdir}/lower_cpu_name.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp > "${outputdir}/lower_sidp.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id > "${outputdir}/lower_id.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp > "${outputdir}/lower_timestamp.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo > "${outputdir}/lower_sopcinfo.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ +} + +return this; \ No newline at end of file From 5127186c4f974c68c635d82a28ef6d7a80be1ee9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 18:26:50 +0000 Subject: [PATCH 264/589] Output version information to meta --- compile/eclipsebuild/eclipseniosbuild.groovy | 55 ++++++++++++-------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy index 18c31d1..a3bb4b9 100644 --- a/compile/eclipsebuild/eclipseniosbuild.groovy +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -76,28 +76,39 @@ def modifyapp(NIOSEDS, bspfolder, wdir) { } } -def dump_elf_info(NIOSEDS, elffile, outputdir) { - sh """#!/bin/bash - set -x - - mkdir -p "${outputdir}" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name > "${outputdir}/lower_cpu_name.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp > "${outputdir}/lower_sidp.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id > "${outputdir}/lower_id.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp > "${outputdir}/lower_timestamp.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo > "${outputdir}/lower_sopcinfo.txt" - if [ \$? -ne 0 ]; then exit 1; fi - """ +def dump_elf_info(NIOSEDS, elffile, meta, custominfokeyprefix) { + if (isUnix()) { + def cpu_name = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_cpu_name", cpu_name) + def sidp = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_sidp", sidp) + def id = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_id", id) + def timestamp = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_timestamp", timestamp) + def sopcinfo = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_sopcinfo", sopcinfo) + } } return this; \ No newline at end of file From a20fce505017824b40b7185b6ee4127e24a92c75 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 18:33:06 +0000 Subject: [PATCH 265/589] Output version information to meta --- compile/eclipsebuild/eclipseniosbuild.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy index a3bb4b9..ce7d798 100644 --- a/compile/eclipsebuild/eclipseniosbuild.groovy +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -79,31 +79,31 @@ def modifyapp(NIOSEDS, bspfolder, wdir) { def dump_elf_info(NIOSEDS, elffile, meta, custominfokeyprefix) { if (isUnix()) { def cpu_name = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_cpu_name", cpu_name) def sidp = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_sidp", sidp) def id = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_id", id) def timestamp = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_timestamp", timestamp) def sopcinfo = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo """, returnStdout: true ) From d1f32dc580e9b5b299fddcc11d0289a780f00b00 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 21:33:59 +0000 Subject: [PATCH 266/589] Output version information to meta --- compile/eclipsebuild/eclipseniosbuild.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy index ce7d798..a808932 100644 --- a/compile/eclipsebuild/eclipseniosbuild.groovy +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -82,32 +82,34 @@ def dump_elf_info(NIOSEDS, elffile, meta, custominfokeyprefix) { "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_cpu_name", cpu_name) def sidp = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_sidp", sidp) def id = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_id", id) def timestamp = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_timestamp", timestamp) +/* def sopcinfo = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_sopcinfo", sopcinfo) +*/ } } From 9fda4bce8b159fcba75abc8cf6a91f3d7d18950e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Dec 2018 13:40:05 +0000 Subject: [PATCH 267/589] Turn off fingerprinting --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 0155c7f..093ddef 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -7,7 +7,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def artifactKey = installerDetails.get(installer)[0] (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy) artifactFetched.put(artifactKey, importmeta) } } From c41563f0c216b19902af393136ce2d4cba468f60 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:33:17 +0000 Subject: [PATCH 268/589] Basebuildnumber is now an external parameter --- versionnumbering/generate.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 7e5568a..f26806d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,6 +136,11 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { + def int bbn = basebuildnumber.trim().toInteger() + return generate(meta, String sourcerevision, String sourcepath, bbn) +} + int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" From f4c573627e3e0a949d04764bf211733892d0c096 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:46:09 +0000 Subject: [PATCH 269/589] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f26806d..e2dc932 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -18,7 +18,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) + adjustedchangecount = ((changecount + basebuildnumber).abs()) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() From db1ebc1959b7bb4818aacc8f79e830cba1431a70 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:50:06 +0000 Subject: [PATCH 270/589] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e2dc932..e28fa2f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,7 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { def int bbn = basebuildnumber.trim().toInteger() - return generate(meta, String sourcerevision, String sourcepath, bbn) + return generate(meta, sourcerevision, sourcepath, bbn) } int incrementrevision(int current, int increment) { From fc6258b158c691c550194a50db86bbd06fb83af4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:55:27 +0000 Subject: [PATCH 271/589] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e28fa2f..4b11cf0 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,7 +136,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } -String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { def int bbn = basebuildnumber.trim().toInteger() return generate(meta, sourcerevision, sourcepath, bbn) } From f7010f5eaf8cb9aeb77ca252f0616b14ceb19b0c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 14:03:46 +0000 Subject: [PATCH 272/589] Set buildstamp --- meta/folder.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 15f77d0..629aa5b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -161,7 +161,7 @@ class BuildStampInfo extends GISJsonSerialised { int revision String buildstamp - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { + BuildStampInfo(script, buildstampprefix = "1.30000.0.", revision = 0, buildstamp = "1.30000.0.0") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision @@ -188,6 +188,12 @@ class BuildStampInfo extends GISJsonSerialised { script.echo("Build stamp now ${buildstamp}") } + void set(String buildstampprefix, int revision) { + buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' + this.buildstampprefix = buildstampprefix + changerevision(revision) + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From f664375cc2bf27ccd89fc672b01d418a4a3c5462 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 14:10:10 +0000 Subject: [PATCH 273/589] Set buildstamp --- meta/folder.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 629aa5b..5ba3485 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -194,6 +194,10 @@ class BuildStampInfo extends GISJsonSerialised { changerevision(revision) } + void set(String buildstampprefix, String revision) { + set(buildstampprefix, revision.toInteger()) + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From 83c89342711ecdc6509d1f5a312990d21b585a4c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 14:21:29 +0000 Subject: [PATCH 274/589] Set buildstamp --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 5ba3485..39de31f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -191,6 +191,7 @@ class BuildStampInfo extends GISJsonSerialised { void set(String buildstampprefix, int revision) { buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' this.buildstampprefix = buildstampprefix + this.buildstamp = buildstampprefix + "0" changerevision(revision) } From fc17af2c5faf83682d17cd9bead0e3f5972c39e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 31 Dec 2018 16:09:25 +0000 Subject: [PATCH 275/589] Handle components that don't have associated source code --- release/do.groovy | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 85753e6..1ded952 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -68,23 +68,25 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } } } } @@ -297,10 +299,15 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" def String buildsha = buildmeta.gitinfo.gitrevision - def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) - def tagadded = sourcegittags(buildsha, srctaglist) - if (tagadded) { - sourcegitpush(dryrunpush) + if (buildsha) { + def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } + else { + summaryappend(",No Src ") } } } From fa1feb73b91d0ac86865e1c8d9bca9d8b0a1f9fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 5 Jan 2019 12:18:09 +0000 Subject: [PATCH 276/589] Automatically add the top level meta item tag to all subcomponents --- release/release.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/release/release.groovy b/release/release.groovy index 828f1ff..3c13368 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -42,6 +42,9 @@ pipeline { steps { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + // Place the top level build stamp on all + def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" + OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason From 67d92ba13af4f6cf4e3143d83efc72e31f3f3a3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Jan 2019 14:13:42 +0000 Subject: [PATCH 277/589] Make other tags component unique --- release/do.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 1ded952..852f6f3 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -157,14 +157,15 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { return changed } -def ArrayList maketaglist(String buildstamp, String otherreleasetags) { +def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { def ArrayList taglist = new ArrayList() taglist.add(buildstamp) def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - taglist.add(othertagitem.trim()) + componentothertagitem = "${othertagitem}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem.trim()) } } return taglist @@ -288,7 +289,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + ArrayList taglist = maketaglist(buildmeta, buildstamp, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { @@ -300,7 +301,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { - def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamp, otherreleasetags) def tagadded = sourcegittags(buildsha, srctaglist) if (tagadded) { sourcegitpush(dryrunpush) From 94b5ef18591b9a26758c93702221dc51d04ebc75 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Jan 2019 14:25:20 +0000 Subject: [PATCH 278/589] Make other tags component unique; Report more info --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 852f6f3..0e53b7f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -32,7 +32,8 @@ def preconditions() { def setunstable(String reason) { markunstable = true - unstablereason = reason + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" } def summaryappend(String summary) { From f4d8ff7a5315f15a6a91ad4eb2fd10c086bd582b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Jan 2019 14:35:37 +0000 Subject: [PATCH 279/589] Make other tags component unique; Report more info --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0e53b7f..0ea3399 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -165,8 +165,8 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - componentothertagitem = "${othertagitem}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" - taglist.add(componentothertagitem.trim()) + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem) } } return taglist From 329997e9829484c8d350d339fca2bd645c594bf4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 12 Jan 2019 12:35:30 +0000 Subject: [PATCH 280/589] On releasing, release the top level component last - so that we don't end up with partial release info --- meta/folder.groovy | 6 +++++- release/do.groovy | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 39de31f..ff56928 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -658,12 +658,16 @@ class MetaInfo extends GISJsonSerialised { return releasemetacollection } + String componentidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + } + Map releasewalkerunique(int maxdepth, String startpath = "") { Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { - String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + String uniquestring = relatedmetainfo.meta.componentidentifier() releasemetauniquecollection.put(uniquestring, relatedmetainfo) } return releasemetauniquecollection diff --git a/release/do.groovy b/release/do.groovy index 0ea3399..e264cc0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,11 +341,18 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) + def toplevelid = buildmeta.componentidentifier() // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - releasecomponent(buildinfo, otherreleasetags, dryrunpush) + if (toplevelid != key) { + releasecomponent(buildinfo, otherreleasetags, dryrunpush) + } } + // Do the top level last + def toplevelbuildinfo = componentReleaseInfo[toplevelid] + releasecomponent(toplevelbuildinfo, otherreleasetags, dryrunpush) + return [summarystring, markunstable, unstablereason] } From 50db2ba7dfbed6d84b61c4679fb34b92b46066c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 07:58:47 +0000 Subject: [PATCH 281/589] Publishing jobs need to provide the meta depth --- publish/publish.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index 8ed7aa8..a703c9e 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -44,7 +44,6 @@ pipeline { steps { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - MAX_META_DEPTH='2' (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason From 59a74b1f9aceda55d7f8a1d1f6f7dbf65b2fb412 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 10:11:51 +0000 Subject: [PATCH 282/589] Do an integrity check at release time; Print out information about components --- publish/do.groovy | 10 ++++++++-- release/do.groovy | 23 +++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index e280c0c..76dc025 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -107,10 +107,16 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin resetfolders() preconditions() - // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = releasemeta.releasewalkerunique(md) + // Walk the meta information to a max depth, looking for artifacts to publish + (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) // For each component + echo "Components in publish:" + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, softwarerelease) diff --git a/release/do.groovy b/release/do.groovy index e264cc0..cbecc08 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -338,8 +338,27 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String resetfolders() preconditions() - // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = buildmeta.releasewalkerunique(md) + // Walk the meta information to a max depth, looking for components to release + (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) + + + echo "Components in release:" + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + + echo "Version unique components in release:" + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + + if (DO_INTEGRITY_CHECK == "true") { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique()) { + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" + } + } def toplevelid = buildmeta.componentidentifier() // For each component From d7834ac5895432b8fdf8ea4e855ec7a70d7d3dac Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 10:36:00 +0000 Subject: [PATCH 283/589] Do an integrity check at release time; Print out information about components --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index cbecc08..a8f9e49 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -339,7 +339,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String preconditions() // Walk the meta information to a max depth, looking for components to release - (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) + (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) echo "Components in release:" From 4b720911b4be5e97c1f9ae0ba5a5088eafaf4bdd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 12:59:29 +0000 Subject: [PATCH 284/589] Do an integrity check at release time; Print out information about components --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index a8f9e49..d1952f0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,7 +341,6 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) - echo "Components in release:" for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] From 4ad0007dceb8155e7d6988c26e7e15e21aaae9f0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:14:57 +0000 Subject: [PATCH 285/589] Do an integrity check at release time; Print out information about components --- publish/do.groovy | 11 ++--------- release/do.groovy | 14 +------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 76dc025..907f328 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -110,15 +110,8 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // Walk the meta information to a max depth, looking for artifacts to publish (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) - // For each component - echo "Components in publish:" - for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" - } - - for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] recallcomponent(releaseinfo, softwarerelease) } stripgithistory() diff --git a/release/do.groovy b/release/do.groovy index d1952f0..b2577fb 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,20 +341,8 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) - echo "Components in release:" - for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" - } - - echo "Version unique components in release:" - for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" - } - if (DO_INTEGRITY_CHECK == "true") { - if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique()) { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" } } From ce93cdb372560f2313802310f84442ac3e168a8a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:40:33 +0000 Subject: [PATCH 286/589] Do an integrity check at release time; Print out information about components --- meta/folder.groovy | 25 ++++++++++++++++++++++--- publish/do.groovy | 8 +++++++- release/do.groovy | 17 ++++++++++++++++- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ff56928..2ef8e22 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -662,17 +662,36 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname } + String componentversionuniqueidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp + } + Map releasewalkerunique(int maxdepth, String startpath = "") { - Map releasemetauniquecollection = new LinkedHashMap() - ArrayList releasemetacollection = new ArrayList() + def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + + def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { - String uniquestring = relatedmetainfo.meta.componentidentifier() + def String uniquestring = relatedmetainfo.meta.componentidentifier() releasemetauniquecollection.put(uniquestring, relatedmetainfo) } + return releasemetauniquecollection } + Map releasewalkerversionunique(int maxdepth, String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + + def Map releasemetaversionuniquecollection = new LinkedHashMap() + for (relatedmetainfo in releasemetacollection) { + def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() + releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + } + + return releasemetaversionuniquecollection + } + MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } diff --git a/publish/do.groovy b/publish/do.groovy index 907f328..a88f018 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -108,7 +108,13 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin preconditions() // Walk the meta information to a max depth, looking for artifacts to publish - (componentReleaseInfo, componentReleaseInfoVersionUnique) = releasemeta.releasewalkerunique(md) + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) + + echo "Version unique components:" + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] diff --git a/release/do.groovy b/release/do.groovy index b2577fb..172861b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -339,7 +339,22 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String preconditions() // Walk the meta information to a max depth, looking for components to release - (componentReleaseInfo, componentReleaseInfoVersionUnique) = buildmeta.releasewalkerunique(md) + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + + script.echo "Unique components:" + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + script.echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) + + echo "Version unique components:" + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + } if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { From 20e86747ef8b0a174c34e15f284e647ecbbc87de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:41:54 +0000 Subject: [PATCH 287/589] Do an integrity check at release time; Print out information about components --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 172861b..0c070d1 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,10 +341,10 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release def componentReleaseInfo = buildmeta.releasewalkerunique(md) - script.echo "Unique components:" + echo "Unique components:" for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - script.echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" } // Walk the meta information to a max depth, looking for components to release From 3567488f00d39277ee8a22736289b62db34b7ae3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 14 Jan 2019 13:51:17 +0000 Subject: [PATCH 288/589] Do an integrity check at release time; Print out information about components --- publish/do.groovy | 5 +++-- release/do.groovy | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index a88f018..570cb5e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -111,10 +111,11 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) echo "Version unique components:" + def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + versionuniquecomponent += "${key}\n" } + echo versionuniquecomponent for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] diff --git a/release/do.groovy b/release/do.groovy index 0c070d1..2326ad5 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -340,21 +340,21 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for components to release def componentReleaseInfo = buildmeta.releasewalkerunique(md) - echo "Unique components:" + def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { - def releaseinfo = componentReleaseInfo[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + versioncomponent += "${key}\n" } + echo versioncomponent // Walk the meta information to a max depth, looking for components to release def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) - echo "Version unique components:" + def String versionuniquecomponent = "" for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - echo "${releaseinfo.offsetpath} : ${releaseinfo.meta.componentversionuniqueidentifier()}" + versionuniquecomponent += "${key}\n" } + echo versionuniquecomponent if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { From 42df36426c1496d2e67162e9519dfdab542dea0b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 21 Jan 2019 15:15:27 +0000 Subject: [PATCH 289/589] Handle missing jq command --- differences/json.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/differences/json.groovy b/differences/json.groovy index 47d9cc5..19b7cf6 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -3,6 +3,8 @@ def comparefiles(importedjsonfile, currentjsonfile) { env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ + which jq + if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") """ From 4fd07d22d652a396fab26ef6f468ad509786faf8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Jan 2019 15:32:00 +0000 Subject: [PATCH 290/589] Handle the case where rebuilt binaries change even though the original source has not changed and the version number has not changed - Tags now have the build number appended to them so that there is no tag clash during the release process --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2326ad5..79bafd5 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -270,7 +270,7 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp + def String buildstamp = "${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() @@ -298,7 +298,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" + def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamp}" def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { From ca173c1f1310b1e743e626b4b2784b3eef3dbef2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Jan 2019 15:53:13 +0000 Subject: [PATCH 291/589] Handle the case where rebuilt binaries change even though the original source has not changed and the version number has not changed - Tags now have the build number appended to them so that there is no tag clash during the release process - change the meta build tags too --- release/do.groovy | 11 ++++++----- release/release.groovy | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 79bafd5..1a785b3 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -270,7 +270,8 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = "${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" + def String buildstamp = buildmeta.buildstampinfo.buildstamp + def String buildstamptag = "${buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() @@ -280,7 +281,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def boolean changed = checkforchange(buildmeta, previousreleasemeta) if (changed) { - releasegitcommit(buildstamp) + releasegitcommit(buildstamptag) } else { releasegitreset() @@ -290,7 +291,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - ArrayList taglist = maketaglist(buildmeta, buildstamp, otherreleasetags) + ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { @@ -298,11 +299,11 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamp}" + def String srcbuildstamptag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamptag}" def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { - def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamp, otherreleasetags) + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) def tagadded = sourcegittags(buildsha, srctaglist) if (tagadded) { sourcegitpush(dryrunpush) diff --git a/release/release.groovy b/release/release.groovy index 3c13368..b0530be 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,7 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" + def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { From 5c5bc9a97804dcf418a87eff5bb187fdc581239a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 27 Jan 2019 11:11:17 +0000 Subject: [PATCH 292/589] Adjust publish script to handle buildstamps appended with build number --- publish/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 570cb5e..b188165 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -44,7 +44,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def releasemeta = releaseinfo.meta def String componentURL = releasemeta.releaseinfo.releaserepoURL - def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo def ArrayList publishablePaths = new ArrayList() if (publishableitems.size() > 0) { @@ -78,11 +78,11 @@ def publish(String customer) { def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}_${releasemeta.jenkinsbuildinfo.buildnumber}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname} ${releasemeta.jenkinsbuildinfo.buildnumber}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) From 91f0f07e8d4ca0aa3fea5538eb1904e2f92b44ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 27 Jan 2019 11:27:09 +0000 Subject: [PATCH 293/589] Adjust publish script to handle buildstamps appended with build number - The output folders and summary don't need to be keyed by build number --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b188165..7844289 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -78,11 +78,11 @@ def publish(String customer) { def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}_${releasemeta.jenkinsbuildinfo.buildnumber}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname} ${releasemeta.jenkinsbuildinfo.buildnumber}: ") + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) From fb25a6b30b16515306aaa7032dcf94484d82e53c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 27 Jan 2019 12:21:29 +0000 Subject: [PATCH 294/589] When generating revision numbers, take into account negative basebuildnumbers --- versionnumbering/generate.groovy | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 4b11cf0..4761174 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -4,6 +4,12 @@ def int maxversionnumber() { return MAXVERSIONNUMBER } +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" + return newrevision +} + def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount def String gitrevision @@ -18,7 +24,9 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" - adjustedchangecount = ((changecount + basebuildnumber).abs()) % (maxversionnumber() + 1) + def int mvnp1 = maxversionnumber() + 1 + basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber + adjustedchangecount = incrementrevision(changecount, basebuildnumber) echo "Adjusted change count: ${adjustedchangecount}" if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -141,10 +149,4 @@ String generate(meta, String sourcerevision, String sourcepath, String basebuild return generate(meta, sourcerevision, sourcepath, bbn) } -int incrementrevision(int current, int increment) { - def int newrevision = (current + increment) % (maxversionnumber() + 1) - echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" - return newrevision -} - return this; \ No newline at end of file From a80034e809b74e60c5109b7349c8e4af7a53e279 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 11:18:49 +0000 Subject: [PATCH 295/589] Certain components are built from the same source repository so therefore their version numbers should not be incremented when they are imported into a build component using the same source repository --- artifacts/metaexport.groovy | 5 +++-- import/importer.groovy | 12 ++++++++---- install/installhelper.groovy | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0ca5da7..043c4de 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -9,11 +9,12 @@ def combineSubjobArtifacts(meta, subJobs) { if (subJobObject != null) { def subJobName = subJobObject.getProjectName() def subJobNumber = subJobObject.getNumber() - destFolder = subJob[1] + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) artifactFolders.add("${destFolder}/**") } } diff --git a/import/importer.groovy b/import/importer.groovy index 1f89426..5c996b4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,6 +1,6 @@ versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -10,11 +10,13 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -27,7 +29,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 093ddef..03ee9e4 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -5,9 +5,9 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] - (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) artifactFetched.put(artifactKey, importmeta) } } From 18272f5bc97fedf92d6bf70bc23d15d96e778bcf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 22:14:31 +0000 Subject: [PATCH 296/589] Alter location of differences file so that it no longer gets published --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2ef8e22..461a8fb 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -419,6 +419,7 @@ class PublishableInfo extends GISJsonSerialised { class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" + private static String DIFFFOLDERNAME = "diff" private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" @@ -565,7 +566,7 @@ class MetaInfo extends GISJsonSerialised { } static String difffilepath() { - return METAFOLDERNAME + "/" + METADIFFFILENAME + return DIFFFOLDERNAME + "/" + METADIFFFILENAME } MetaInfo importrelated(String destpath) { From a4fc87477f8bb4a9cfaf62feff4b6b14fc281897 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 23:07:18 +0000 Subject: [PATCH 297/589] Alter location of differences file so that it no longer gets published --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 461a8fb..efc8df2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -423,6 +423,7 @@ class MetaInfo extends GISJsonSerialised { private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String DIFFARTIFACTS = "${DIFFFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -621,7 +622,7 @@ class MetaInfo extends GISJsonSerialised { ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { - def String artifactlist = METAARTIFACTS + def String artifactlist = "${METAARTIFACTS},${DIFFARTIFACTS}" releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { artifactlist += "," + releaseitem From 54185b50486c7eef935aabbe62e77f14cabe2dfc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 31 Jan 2019 18:32:44 +0000 Subject: [PATCH 298/589] Select the artifacts from a specific build for the duration of the job --- meta/folder.groovy | 4 ++-- release/do.groovy | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index efc8df2..1b74fed 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -118,8 +118,8 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { - private String buildjobname - private int buildnumber + String buildjobname + int buildnumber private String buildmachine private String builddate diff --git a/release/do.groovy b/release/do.groovy index 1a785b3..1d9e721 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -92,7 +92,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { } } -def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { @@ -104,10 +104,10 @@ def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { echo "Copying artifacts: ${filterpath} to ${tempfolder}" copyArtifacts( filter: filterpath, - projectName: BUILD_JOB, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR + parameterName: " ${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}") ], target: tempfolder ) @@ -315,7 +315,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } -def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { +def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { def buildmeta = buildinfo.meta def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" @@ -327,7 +327,7 @@ def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp) gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) @@ -368,12 +368,12 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] if (toplevelid != key) { - releasecomponent(buildinfo, otherreleasetags, dryrunpush) + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) } } // Do the top level last def toplevelbuildinfo = componentReleaseInfo[toplevelid] - releasecomponent(toplevelbuildinfo, otherreleasetags, dryrunpush) + releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) return [summarystring, markunstable, unstablereason] } From 4b6c3aff23174460d121db08793cb9a586862976 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 31 Jan 2019 18:38:21 +0000 Subject: [PATCH 299/589] Select the artifacts from a specific build for the duration of the job --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1d9e721..0f65035 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -107,7 +107,7 @@ def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, Strin projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: " ${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}") + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" ], target: tempfolder ) From 628fe40745113bb353cb60021fc0f61ce527321e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 31 Jan 2019 22:35:18 +0000 Subject: [PATCH 300/589] Clone depth 1 is causing issues during release to git --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 0f65035..604581a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -57,7 +57,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + sh("git clone ${releaserepo} ${checkoutfolder}") } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { From 05472dfcda37b37b20ed1f79c9fed2c06eb29cba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 11:36:23 +0000 Subject: [PATCH 301/589] Update build script to use updated Common_scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1b74fed..934ffe3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,7 +157,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - private String buildstampprefix + String buildstampprefix int revision String buildstamp From 7323493b05ab01ff46cf1d86207dfc19ff761870 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:28:15 +0000 Subject: [PATCH 302/589] Add non-build startup boilerplate --- autobuilder.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..1210aab 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -56,6 +56,28 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA return meta } +// Start up boilderplate +def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = getmetafolderscript().metareset(script) + meta.productinfo.set("", "", "") + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } From eb07ef6dc046a36c85c16ea32799a756a7eda3b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:43:08 +0000 Subject: [PATCH 303/589] Try to use the REPO_ROOT environment variable --- versioncontrol/git.groovy | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..730335a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,12 +19,8 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" - } else { - reporootpath = "" +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = env.REPO_ROOT.trim(), int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + if (repoRoot == "") { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } checkout([ @@ -39,7 +35,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: repoRoot, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], sparsecheckoutconfig(sparseitemlist) From 31196dbede6b5c339c21dfb24080e928bf4bd435 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 18:47:04 +0000 Subject: [PATCH 304/589] Revert "Try to use the REPO_ROOT environment variable" This reverts commit aa04eb23b85ac42d236d21dc489f3a03038fbfba. --- versioncontrol/git.groovy | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 730335a..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,8 +19,12 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = env.REPO_ROOT.trim(), int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - if (repoRoot == "") { +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } checkout([ @@ -35,7 +39,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: repoRoot, shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], sparsecheckoutconfig(sparseitemlist) From 9f47550d0e051c80f3ef9c44163558dd6b2d5e7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 19:07:43 +0000 Subject: [PATCH 305/589] Try to speed up release repo check out --- autobuilder.groovy | 4 ++-- release/registerrepo.groovy | 4 ++-- versioncontrol/git.groovy | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 1210aab..b9a36d4 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -32,7 +32,7 @@ def getmetafolderscript() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -48,7 +48,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def rr = load("${commonscripts()}/release/registerrepo.groovy") - rr.registerrepo(meta, releaserepo) + rr.registerrepo(meta, releaserepo, repoRootSuffix) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index ff4d85d..2dffdc7 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -2,9 +2,9 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo) { +def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix) } meta.releaseinfo.set(releaserepo) } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..f7d10bb 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,10 +19,11 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" + if (reporootenv != null && repoRootSuffix != "") { + reporootpath = "${reporootenv.trim()}/${repoRootSuffix}" + echo "Using release repo: ${reporootpath}" } else { reporootpath = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" From 643161784419d1c66ab10c56e3db6195c3d55778 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 21:02:16 +0000 Subject: [PATCH 306/589] Update build script to use updated Common_scripts - Installers --- install/installhelper.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 03ee9e4..0447cb8 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -4,7 +4,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { - def artifactKey = installerDetails.get(installer)[0] + def artifactKey = installerDetails.get(installer)[1] (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) @@ -17,7 +17,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { // Export any required binaries to the /bin folder def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { - (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { From 2ab7619f1a649a54e465a5dd8729e3e09947c603 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 21:15:21 +0000 Subject: [PATCH 307/589] Update build script to use updated Common_scripts - Installers --- install/installhelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 0447cb8..819a580 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -19,10 +19,10 @@ def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { - (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { - binariesSourceFolder = "${destinationFolder}/${binaryFolder}" - binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + def binariesSourceFolder = "${destFolder}/${binaryFolder}" + def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) } From 134f5497362b537147699257f3ac79322ca9ff61 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 11 Feb 2019 23:14:18 +0000 Subject: [PATCH 308/589] Update build script to use updated Common_scripts - Installers --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index b9a36d4..63ca2de 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -67,7 +67,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("", "", "") + meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) From 3264655356b72f27aa28e5fbd4ccb5a77a47efd3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 18:20:05 +0000 Subject: [PATCH 309/589] Compare hps_0.h files --- differences/file.groovy | 12 ++++++++++++ differences/json.groovy | 4 +--- 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 differences/file.groovy diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..f5e93da --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,12 @@ +def comparefiles(importedfile, currentfile) { + if (isUnix()) { + sh """ + diff -c <"${importedfile}" <"${currentfile}" + """ + } + else { + error "Windows not currently supporting file comparison" + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy index 19b7cf6..5ac37f2 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -1,12 +1,10 @@ def comparefiles(importedjsonfile, currentjsonfile) { - env.IMPORTED_JSON="${importedjsonfile}" - env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ which jq if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output - diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") """ } else { From 2013c582c73ae5adf5b1c0bc26da48963faac83a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 19:01:59 +0000 Subject: [PATCH 310/589] Compare hps_0.h files --- differences/file.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/file.groovy b/differences/file.groovy index f5e93da..dee78b9 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,7 +1,7 @@ def comparefiles(importedfile, currentfile) { if (isUnix()) { sh """ - diff -c <"${importedfile}" <"${currentfile}" + diff -c "${importedfile}" "${currentfile}" """ } else { From 06c096fc7c054611d8847394e52bf1916163633b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Feb 2019 19:17:49 +0000 Subject: [PATCH 311/589] Compare hps_0.h files --- differences/file.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/file.groovy b/differences/file.groovy index dee78b9..ab4102f 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,7 +1,11 @@ -def comparefiles(importedfile, currentfile) { +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { if (isUnix()) { + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } sh """ - diff -c "${importedfile}" "${currentfile}" + diff -c ${wsparameter} "${importedfile}" "${currentfile}" """ } else { From f61e44d6dbe4ddfb39935d26dd14eb6edaf7a251 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 15 Feb 2019 13:42:58 +0000 Subject: [PATCH 312/589] Debug publishable list addition --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 934ffe3..52ceb06 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -486,6 +486,7 @@ class MetaInfo extends GISJsonSerialised { void addPublishable(String filepath) { def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") publishableinfo.add(pi) } From 0f636ea4276e66e4656eaf8afb2e9ae760abba27 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 16:07:52 +0000 Subject: [PATCH 313/589] Try to create WORKSPACE relative paths --- meta/folder.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 52ceb06..245d961 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -554,13 +554,16 @@ class MetaInfo extends GISJsonSerialised { } String relativize(String path) { - def Path thispath = Paths.get(path) - if (thispath.startsWith(script.WORKSPACE)) { + def Path mypath = Paths.get(path) + String pathRelative + if (mypath.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(thispath) - return wsrelativePath.toString() + def Path wsrelativePath = wspath.relativize(mypath) + pathRelative = wsrelativePath.toString() + } else { + pathRelative = mypath.toString() } - return path + return pathRelative } static String metafilepath() { From dc6e7c6b1d5a0e54aec98ce61539ae27b855c2dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:05:25 +0000 Subject: [PATCH 314/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 245d961..d7cbc4f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -556,12 +556,15 @@ class MetaInfo extends GISJsonSerialised { String relativize(String path) { def Path mypath = Paths.get(path) String pathRelative - if (mypath.startsWith(script.WORKSPACE)) { + script.echo("Seeing if ${path} begins with from ${script.WORKSPACE}") + if (path.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) pathRelative = wsrelativePath.toString() + script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}" } else { pathRelative = mypath.toString() + script.echo("Path relative is unchanged: ${pathRelative}" } return pathRelative } From de9e4d82ed9664de567db925c1213be31cc8e84a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:07:33 +0000 Subject: [PATCH 315/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d7cbc4f..0c94a12 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -561,7 +561,7 @@ class MetaInfo extends GISJsonSerialised { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) pathRelative = wsrelativePath.toString() - script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}" + script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") } else { pathRelative = mypath.toString() script.echo("Path relative is unchanged: ${pathRelative}" From 6ca84a13fecedfda50a8a8f594bf761b60a26fc9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:18:29 +0000 Subject: [PATCH 316/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0c94a12..f6797a2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,7 +564,7 @@ class MetaInfo extends GISJsonSerialised { script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") } else { pathRelative = mypath.toString() - script.echo("Path relative is unchanged: ${pathRelative}" + script.echo("Path relative is unchanged: ${pathRelative}") } return pathRelative } From d66b7b80d602550ccae4aca1b1483f36f19b4a67 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:21:14 +0000 Subject: [PATCH 317/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f6797a2..ce041d0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -556,7 +556,7 @@ class MetaInfo extends GISJsonSerialised { String relativize(String path) { def Path mypath = Paths.get(path) String pathRelative - script.echo("Seeing if ${path} begins with from ${script.WORKSPACE}") + script.echo("Seeing if ${path} begins with ${script.WORKSPACE}") if (path.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) From 736f0ece675736032cc8b1dc1f33c313015648e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:45:39 +0000 Subject: [PATCH 318/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ce041d0..74c00cc 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -553,17 +553,14 @@ class MetaInfo extends GISJsonSerialised { deserialiseJson(rawmeta) } - String relativize(String path) { - def Path mypath = Paths.get(path) + String relativize(String mypath) { String pathRelative - script.echo("Seeing if ${path} begins with ${script.WORKSPACE}") - if (path.startsWith(script.WORKSPACE)) { - def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(mypath) - pathRelative = wsrelativePath.toString() - script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") } else { - pathRelative = mypath.toString() + pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } return pathRelative From d2bd59bfa5c3a6d922a118e86fc46e707f4b5f90 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 18:07:16 +0000 Subject: [PATCH 319/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 74c00cc..1f90c61 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -563,6 +563,13 @@ class MetaInfo extends GISJsonSerialised { pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } + // Strip leading / if any + pathRelative = pathRelative.replaceAll("^/+", "") + // Strip leading \ if any + pathRelative = pathRelative.replaceAll("^\\+", "") + + script.echo("Path relative is ${pathRelative}") + return pathRelative } From 4c7726123e43370c5c6397b678615717ec11e8ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:06:38 +0000 Subject: [PATCH 320/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1f90c61..3a4d464 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,9 +564,11 @@ class MetaInfo extends GISJsonSerialised { script.echo("Path relative is unchanged: ${pathRelative}") } // Strip leading / if any - pathRelative = pathRelative.replaceAll("^/+", "") + pathRelative = pathRelative.replaceAll('^/+', '') // Strip leading \ if any - pathRelative = pathRelative.replaceAll("^\\+", "") + pathRelative = pathRelative.replaceAll('^\\+', '') + // Convert \ with / + pathRelative = pathRelative.replaceAll('\\', '/') script.echo("Path relative is ${pathRelative}") From da35fd3aa07e1ee84822752e74a9b2fcc3a5ec3e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:23:02 +0000 Subject: [PATCH 321/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 3a4d464..adfee97 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -568,7 +568,7 @@ class MetaInfo extends GISJsonSerialised { // Strip leading \ if any pathRelative = pathRelative.replaceAll('^\\+', '') // Convert \ with / - pathRelative = pathRelative.replaceAll('\\', '/') + pathRelative = pathRelative.replace('\\', '/') script.echo("Path relative is ${pathRelative}") From 06ca80c596d36ce9c758edd6982132eec3e3c44a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:46:23 +0000 Subject: [PATCH 322/589] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index adfee97..f9d4bf3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -563,12 +563,10 @@ class MetaInfo extends GISJsonSerialised { pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') // Strip leading / if any pathRelative = pathRelative.replaceAll('^/+', '') - // Strip leading \ if any - pathRelative = pathRelative.replaceAll('^\\+', '') - // Convert \ with / - pathRelative = pathRelative.replace('\\', '/') script.echo("Path relative is ${pathRelative}") From a7da6556d91007e844760c552e804cce87f312ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 00:01:14 +0000 Subject: [PATCH 323/589] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 42 ++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..d333af3 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -28,22 +28,50 @@ def doKillLeftovers() { } } +def doCheck(checkpathlist, build_variant = "") { + def result = 0 + for (checkpath in checkpathlist) { + echo "Checking file at: ${checkpath}" + script { + echo "Build variant is ${build_variant}" + if (build_variant) { + def srcFile = "${WORKSPACE}/${checkpath}" + def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + result = sh( script: """ + // Print out dependencies of the built file + readelf -d "${srcFileExpanded}" | grep NEEDED + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + if (result != 0) { + return result + } + } + } + } + return result +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 + def destFolder = "" + def srcFile = "" + def srcFileExpanded = "" for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + destFolder = "${exportfolder}/${build_variant}" + srcFile = "${WORKSPACE}/${exportpath}" + srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" - env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + destFolder = "${exportfolder}/${project}" + srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { @@ -162,6 +190,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } + result = doCheck(exportpathlist, build_variant) + if (result != 0) { + error "Failed to check files...." + } result = doExport(exportfolder, exportpathlist, "", build_variant) if (result != 0) { error "Failed to copy...." From 4c7053017eadcc4d6fe308bf6abbe087de31f0dc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 00:33:36 +0000 Subject: [PATCH 324/589] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index d333af3..2dbd6ea 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -74,7 +74,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { srcFileExpanded = "${WORKSPACE}/${exportpath}" } } - dir (env.destFolder) { + dir (destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { result = sh( script: """ @@ -88,10 +88,10 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } } else { - result = bat( script: ''' - copy \"%srcFileExpanded%\" \"%destFolder%\" + result = bat( script: """ + copy "${srcFileExpanded}" "${destFolder}" if errorlevel 1 EXIT /b 1 - ''', + """, returnStatus: true ) if (result != 0) { From 49f1b99e477c945156e9caaa4215ff8659a360eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 07:55:15 +0000 Subject: [PATCH 325/589] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 2dbd6ea..302b4f8 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -37,8 +37,9 @@ def doCheck(checkpathlist, build_variant = "") { if (build_variant) { def srcFile = "${WORKSPACE}/${checkpath}" def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + // Print out dependencies of the built file + echo "Showing dependencies of ${srcFileExpanded}" result = sh( script: """ - // Print out dependencies of the built file readelf -d "${srcFileExpanded}" | grep NEEDED if [ \$? -ne 0 ]; then exit 1; fi """, From c4247d45ed5bfcfeab751fdb6595158f51c14bdc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 18:37:55 +0000 Subject: [PATCH 326/589] Add publish item selector script --- publish/publish_item_select.groovy | 90 ++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100755 publish/publish_item_select.groovy diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy new file mode 100755 index 0000000..747b9f6 --- /dev/null +++ b/publish/publish_item_select.groovy @@ -0,0 +1,90 @@ +#!/usr/local/bin/groovy +metarepo="git@hydra:jenkinsautobuild/Common_scripts.git" + +import hudson.model.* + +def log(message) { + boolean dolog = true + if (dolog) { + println message + } +} + +println "hello" + +def createfolder(String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + log("Creating directory: ${folder}") + def s = ['mkdir', '-p', folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def gitCloneOrPull(String repo, String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + if (!new File(folder).exists()) { + log("Cloning: ${repo} to: ${folder}") + createfolder(folder) + def s = ['git', 'clone', repo, folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } else { + log("Pull in: ${folder}") + def s = ['/bin/sh', '-c', "cd ${folder}"].execute() | ['git', 'pull'].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def String makeworking() { + def r = ['whoami'].execute() + r.waitFor() + def user = r.text.trim() + + log("User is ${user}") + def rootfolder = "/home/${user}/tmp" + createfolder(rootfolder) + + def commonscripts = "${rootfolder}/commonscripts" + gitCloneOrPull(metarepo, commonscripts); + +} + +def List getItems() { + def List items = new ArrayList() + try { + log("Getting items") + def p = ['pwd'].execute() + p.waitFor() + items.add(p.text) + items.add(metarepo) + items.add(metalevels) + + items.add("ATLAS_SERVICES CORE_INSTALL installers/**") + + items.add("Item 2") + items.add("Item 3") + return items + } catch (Exception e) { + items.add("Error ${e}") + return items + } +} + +makeworking() +count = 0 +for (item in getItems()) { + log (count) + count += 1 + log (item) +} \ No newline at end of file From 07c3e7ce16f61f387e878f610c76da079bf79ae0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 18:51:12 +0000 Subject: [PATCH 327/589] Add publish item selector script --- publish/publish_item_select.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 747b9f6..50090a8 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -29,7 +29,7 @@ def gitCloneOrPull(String repo, String folder) { def sout = new StringBuilder() def serr = new StringBuilder() - if (!new File(folder).exists()) { + if (!(new File(folder).exists())) { log("Cloning: ${repo} to: ${folder}") createfolder(folder) def s = ['git', 'clone', repo, folder].execute() @@ -37,7 +37,7 @@ def gitCloneOrPull(String repo, String folder) { s.waitFor() } else { log("Pull in: ${folder}") - def s = ['/bin/sh', '-c', "cd ${folder}"].execute() | ['git', 'pull'].execute() + def s = ["/bin/sh -c cd ${folder}; git pull"].execute() s.consumeProcessOutput(sout, serr) s.waitFor() } From fdff1c1ef7f568673752e35d1db2c138d377a9f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 18:55:44 +0000 Subject: [PATCH 328/589] Add publish item selector script --- publish/publish_item_select.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 50090a8..68215ff 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -37,7 +37,7 @@ def gitCloneOrPull(String repo, String folder) { s.waitFor() } else { log("Pull in: ${folder}") - def s = ["/bin/sh -c cd ${folder}; git pull"].execute() + def s = ["/bin/bash -c cd ${folder}; pwd; git pull"].execute() s.consumeProcessOutput(sout, serr) s.waitFor() } From d7a8dd14add18cd947df7e4dac99331efe898172 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 19:28:47 +0000 Subject: [PATCH 329/589] Add publish item selector script --- publish/publish_item_select.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 68215ff..56f7e37 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -37,7 +37,7 @@ def gitCloneOrPull(String repo, String folder) { s.waitFor() } else { log("Pull in: ${folder}") - def s = ["/bin/bash -c cd ${folder}; pwd; git pull"].execute() + def s = ['git', 'pull'].execute(null, new File(folder)) s.consumeProcessOutput(sout, serr) s.waitFor() } From 6cc9f1928d2dc28693bbc10f1165b80a3724b688 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Mar 2019 23:34:08 +0000 Subject: [PATCH 330/589] Add publish item selector script --- publish/publish_item_select.groovy | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 56f7e37..ef39d78 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,5 +1,7 @@ #!/usr/local/bin/groovy -metarepo="git@hydra:jenkinsautobuild/Common_scripts.git" +metarepoext = "git@hydra:HMB_XR_5601_Releases/hmb_xr_5601_whole.git" + +commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" import hudson.model.* @@ -46,6 +48,15 @@ def gitCloneOrPull(String repo, String folder) { return [sout, serr] } +def loadmetaclass(String commonscripts) { + def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" + log("Loading script: ${metafolderscriptfile}") + def script = new GroovyScriptEngine('.').with { + loadScriptByName(metafolderscriptfile) + } + return script +} + def String makeworking() { def r = ['whoami'].execute() r.waitFor() @@ -56,8 +67,10 @@ def String makeworking() { createfolder(rootfolder) def commonscripts = "${rootfolder}/commonscripts" - gitCloneOrPull(metarepo, commonscripts); + gitCloneOrPull(commonscriptsrepo, commonscripts); + def metascript = loadmetaclass(commonscripts) + return metascript } def List getItems() { @@ -67,7 +80,7 @@ def List getItems() { def p = ['pwd'].execute() p.waitFor() items.add(p.text) - items.add(metarepo) + items.add(metarepoext) items.add(metalevels) items.add("ATLAS_SERVICES CORE_INSTALL installers/**") @@ -82,6 +95,7 @@ def List getItems() { } makeworking() +loadmetaclass() count = 0 for (item in getItems()) { log (count) From 39d54ecfa090ffa7d6f69a97c4b5ca66ccacfca2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 11:17:25 +0000 Subject: [PATCH 331/589] Add publish item selector script --- meta/folder.groovy | 15 +++++++++- publish/publish_item_select.groovy | 48 ++++++++++++++++++++++-------- 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f9d4bf3..9f747ad 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ import java.nio.file.Paths import java.nio.file.Path +import groovy.json.JsonSlurperClassic abstract class GISJsonSerialised implements Serializable { def script @@ -552,7 +553,13 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } - + + def deserialiseJsonSlurper(String filepath) { + def jsonfile = readFile(file: filepath) + def rawmeta = new JsonSlurperClassic().parseText(jsonfile) + deserialiseJson(rawmeta) + } + String relativize(String mypath) { String pathRelative script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") @@ -726,4 +733,10 @@ MetaInfo metafromfolder(String folder = "") { return buildmeta } +MetaInfo metafromfolderSlurper(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + return this; diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index ef39d78..7eeae29 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,5 +1,7 @@ #!/usr/local/bin/groovy +jobname="anything" metarepoext = "git@hydra:HMB_XR_5601_Releases/hmb_xr_5601_whole.git" +metalevels = 2 commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" @@ -48,13 +50,25 @@ def gitCloneOrPull(String repo, String folder) { return [sout, serr] } -def loadmetaclass(String commonscripts) { +def loadmetaclass(String commonscripts, String metafolder) { def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" log("Loading script: ${metafolderscriptfile}") - def script = new GroovyScriptEngine('.').with { - loadScriptByName(metafolderscriptfile) - } - return script + + GroovyShell shell = new GroovyShell() + def metafolderscript = shell.parse(new File(metafolderscriptfile)) + log("metafolderscript is ${metafolderscript.getClass()}") + def meta = metafolderscript.metafromfolder(metafolder) + log("meta is ${meta.getClass()}") + + + + //Class groovyClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(metafolderscriptfile) + //log("groovyClass is ${groovyClass.getClass()}") + //GroovyObject metafolderobject = (GroovyObject) groovyClass.newInstance() + //log("metafolderobject is ${metafolderobject.getClass()}") + //metaObject = new metafolderobject.MetaInfo(this) + //log("metaobject is ${metaObject.getClass()}") + return meta } def String makeworking() { @@ -67,10 +81,19 @@ def String makeworking() { createfolder(rootfolder) def commonscripts = "${rootfolder}/commonscripts" - gitCloneOrPull(commonscriptsrepo, commonscripts); + gitCloneOrPull(commonscriptsrepo, commonscripts) - def metascript = loadmetaclass(commonscripts) - return metascript + // Clone or pull the relevant meta repo + def metareporoot = "${rootfolder}/${jobname}" + gitCloneOrPull(metarepoext, metareporoot) + def metafolder = "${metareporoot}/meta" + + // Load the meta folder class that interprets the meta repo + def metascript = loadmetaclass(commonscripts, metafolder) + + // Read in the meta + meta = metascript.metafromfolder(metafolder) + return meta } def List getItems() { @@ -79,7 +102,7 @@ def List getItems() { log("Getting items") def p = ['pwd'].execute() p.waitFor() - items.add(p.text) + items.add(p.text.trim()) items.add(metarepoext) items.add(metalevels) @@ -95,10 +118,9 @@ def List getItems() { } makeworking() -loadmetaclass() count = 0 -for (item in getItems()) { - log (count) +def items = getItems() +for (item in items) { count += 1 - log (item) + log ("${count}: ${item}") } \ No newline at end of file From 6746c4dccf1ca52ad77424e1917a989880107542 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 11:24:26 +0000 Subject: [PATCH 332/589] Add publish item selector script --- meta/folder.groovy | 5 +++-- publish/publish_item_select.groovy | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9f747ad..d751f5e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -555,8 +555,9 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJsonSlurper(String filepath) { - def jsonfile = readFile(file: filepath) - def rawmeta = new JsonSlurperClassic().parseText(jsonfile) + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) deserialiseJson(rawmeta) } diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 7eeae29..4af7c2b 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -57,7 +57,7 @@ def loadmetaclass(String commonscripts, String metafolder) { GroovyShell shell = new GroovyShell() def metafolderscript = shell.parse(new File(metafolderscriptfile)) log("metafolderscript is ${metafolderscript.getClass()}") - def meta = metafolderscript.metafromfolder(metafolder) + def meta = metafolderscript.metafromfolderSlurper(metafolder) log("meta is ${meta.getClass()}") From 875e21b092e721f5e35183da36a11832d57c18da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 11:53:57 +0000 Subject: [PATCH 333/589] Add publish item selector script --- publish/publish_item_select.groovy | 53 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 4af7c2b..f02b800 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -60,18 +60,10 @@ def loadmetaclass(String commonscripts, String metafolder) { def meta = metafolderscript.metafromfolderSlurper(metafolder) log("meta is ${meta.getClass()}") - - - //Class groovyClass = new GroovyClassLoader(getClass().getClassLoader()).parseClass(metafolderscriptfile) - //log("groovyClass is ${groovyClass.getClass()}") - //GroovyObject metafolderobject = (GroovyObject) groovyClass.newInstance() - //log("metafolderobject is ${metafolderobject.getClass()}") - //metaObject = new metafolderobject.MetaInfo(this) - //log("metaobject is ${metaObject.getClass()}") return meta } -def String makeworking() { +def makeworking() { def r = ['whoami'].execute() r.waitFor() def user = r.text.trim() @@ -89,37 +81,46 @@ def String makeworking() { def metafolder = "${metareporoot}/meta" // Load the meta folder class that interprets the meta repo - def metascript = loadmetaclass(commonscripts, metafolder) - - // Read in the meta - meta = metascript.metafromfolder(metafolder) + def meta = loadmetaclass(commonscripts, metareporoot) + log("2: meta is ${meta.getClass()}") + return meta } -def List getItems() { - def List items = new ArrayList() +def List getItems(meta) { + def List menuitems = new ArrayList() + def metaitems = meta.releasewalkerunique(metalevels) + log("metaitems is ${metaitems.getClass()}") + try { log("Getting items") def p = ['pwd'].execute() p.waitFor() - items.add(p.text.trim()) - items.add(metarepoext) - items.add(metalevels) + menuitems.add(p.text.trim()) + menuitems.add(metarepoext) + menuitems.add(metalevels) - items.add("ATLAS_SERVICES CORE_INSTALL installers/**") + menuitems.add("ATLAS_SERVICES CORE_INSTALL installers/**") - items.add("Item 2") - items.add("Item 3") - return items + menuitems.add("Item 2") + menuitems.add("Item 3") + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + menuitems.add(releaseinfo.offsetpath) + //MetaInfo meta + } + return menuitems } catch (Exception e) { - items.add("Error ${e}") - return items + menuitems.add("Error ${e}") + return menuitems } } -makeworking() +def meta = makeworking() +log("3: meta is ${meta.getClass()}") + count = 0 -def items = getItems() +def items = getItems(meta) for (item in items) { count += 1 log ("${count}: ${item}") From b480d4a965d99e1463a7f96046b36bcf7491facf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Mar 2019 12:40:37 +0000 Subject: [PATCH 334/589] Add publish item selector script --- publish/publish_item_select.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index f02b800..47b64dc 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,6 +1,6 @@ #!/usr/local/bin/groovy -jobname="anything" -metarepoext = "git@hydra:HMB_XR_5601_Releases/hmb_xr_5601_whole.git" +jobname="anythingnew" +metarepoext = "git@hydra:print-server-releases/print_server_whole.git" metalevels = 2 commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" @@ -124,4 +124,6 @@ def items = getItems(meta) for (item in items) { count += 1 log ("${count}: ${item}") -} \ No newline at end of file +} + +return items \ No newline at end of file From 18f6b42a56f11aee83ea1d62b0a8d013d70782fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 3 Mar 2019 18:02:51 +0000 Subject: [PATCH 335/589] Add publish item selector script; Add eclipse groovy project --- publish/.classpath | 9 +++++++ publish/.gitignore | 7 ++++++ publish/.project | 18 ++++++++++++++ .../org.eclipse.jdt.groovy.core.prefs | 3 +++ publish/publish_item_select.groovy | 24 +------------------ 5 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 publish/.classpath create mode 100644 publish/.gitignore create mode 100644 publish/.project create mode 100644 publish/.settings/org.eclipse.jdt.groovy.core.prefs diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 47b64dc..a7cfa1b 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -5,8 +5,6 @@ metalevels = 2 commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" -import hudson.model.* - def log(message) { boolean dolog = true if (dolog) { @@ -52,14 +50,10 @@ def gitCloneOrPull(String repo, String folder) { def loadmetaclass(String commonscripts, String metafolder) { def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" - log("Loading script: ${metafolderscriptfile}") GroovyShell shell = new GroovyShell() def metafolderscript = shell.parse(new File(metafolderscriptfile)) - log("metafolderscript is ${metafolderscript.getClass()}") def meta = metafolderscript.metafromfolderSlurper(metafolder) - log("meta is ${meta.getClass()}") - return meta } @@ -68,7 +62,6 @@ def makeworking() { r.waitFor() def user = r.text.trim() - log("User is ${user}") def rootfolder = "/home/${user}/tmp" createfolder(rootfolder) @@ -82,8 +75,6 @@ def makeworking() { // Load the meta folder class that interprets the meta repo def meta = loadmetaclass(commonscripts, metareporoot) - log("2: meta is ${meta.getClass()}") - return meta } @@ -93,21 +84,9 @@ def List getItems(meta) { log("metaitems is ${metaitems.getClass()}") try { - log("Getting items") - def p = ['pwd'].execute() - p.waitFor() - menuitems.add(p.text.trim()) - menuitems.add(metarepoext) - menuitems.add(metalevels) - - menuitems.add("ATLAS_SERVICES CORE_INSTALL installers/**") - - menuitems.add("Item 2") - menuitems.add("Item 3") for (key in metaitems.keySet()) { def releaseinfo = metaitems[key] menuitems.add(releaseinfo.offsetpath) - //MetaInfo meta } return menuitems } catch (Exception e) { @@ -117,10 +96,9 @@ def List getItems(meta) { } def meta = makeworking() -log("3: meta is ${meta.getClass()}") +def items = getItems(meta) count = 0 -def items = getItems(meta) for (item in items) { count += 1 log ("${count}: ${item}") From c4a5e7a36177d7218b0af68c618bb9122516360c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 3 Mar 2019 22:24:58 +0000 Subject: [PATCH 336/589] Add publish item selector script; Add eclipse groovy project --- publish/publish_item_select.groovy | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index a7cfa1b..2299f37 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,19 +1,18 @@ #!/usr/local/bin/groovy jobname="anythingnew" metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +outputselect = "description" metalevels = 2 +dolog = false commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" def log(message) { - boolean dolog = true if (dolog) { println message } } -println "hello" - def createfolder(String folder) { def sout = new StringBuilder() def serr = new StringBuilder() @@ -78,6 +77,22 @@ def makeworking() { return meta } +def String outputDescription(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } +} + +def String outputSelection(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } +} + def List getItems(meta) { def List menuitems = new ArrayList() def metaitems = meta.releasewalkerunique(metalevels) @@ -85,8 +100,13 @@ def List getItems(meta) { try { for (key in metaitems.keySet()) { - def releaseinfo = metaitems[key] - menuitems.add(releaseinfo.offsetpath) + def releaseinfo = metaitems[key] + if (outputselect == "description") { + outputDescription(menuitems, releaseinfo) + } + if (outputselect == "selection") { + outputSelection(menuitems, releaseinfo) + } } return menuitems } catch (Exception e) { From 2426465d7db8aab7801a86e1e6c1bbd105baabc7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 3 Mar 2019 22:57:57 +0000 Subject: [PATCH 337/589] Add publish item selector script --- publish/publish_item_select.groovy | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy index 2299f37..99e424f 100755 --- a/publish/publish_item_select.groovy +++ b/publish/publish_item_select.groovy @@ -1,8 +1,16 @@ #!/usr/local/bin/groovy -jobname="anythingnew" -metarepoext = "git@hydra:print-server-releases/print_server_whole.git" -outputselect = "description" -metalevels = 2 + +jobnameext = jobname ?: "jobunique" +metarepoext = metarepo ?: "git@hydra:print-server-releases/print_server_whole.git" +outputselectext = outputselect ?: "description" +metalevelsstringext = metalevels ?: "2" + +//jobnameext = "jobunique" +//metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +//outputselectext = "description" +//metalevelsstringext = "2" + +metalevelsext = metalevelsstringext.toInteger() dolog = false commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" @@ -68,7 +76,7 @@ def makeworking() { gitCloneOrPull(commonscriptsrepo, commonscripts) // Clone or pull the relevant meta repo - def metareporoot = "${rootfolder}/${jobname}" + def metareporoot = "${rootfolder}/${jobnameext}" gitCloneOrPull(metarepoext, metareporoot) def metafolder = "${metareporoot}/meta" @@ -95,16 +103,16 @@ def String outputSelection(menuitems, releaseinfo) { def List getItems(meta) { def List menuitems = new ArrayList() - def metaitems = meta.releasewalkerunique(metalevels) + def metaitems = meta.releasewalkerunique(metalevelsext) log("metaitems is ${metaitems.getClass()}") try { for (key in metaitems.keySet()) { def releaseinfo = metaitems[key] - if (outputselect == "description") { + if (outputselectext == "description") { outputDescription(menuitems, releaseinfo) } - if (outputselect == "selection") { + if (outputselectext == "selection") { outputSelection(menuitems, releaseinfo) } } From bb65e72373fe39938ef6f96b13128db3c53b7e77 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 13 Mar 2019 18:57:10 +0000 Subject: [PATCH 338/589] Clone jobs --- versionnumbering/generate.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 4761174..f4f1bdc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -56,7 +56,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // Handle: // origin/master // origin/2.0.x -def getVersionPrefix(String sourcebranch) { +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -121,6 +121,9 @@ def getVersionPrefix(String sourcebranch) { versionprefix += "0." } } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } echo "Determined version prefix to be: ${versionprefix}" return versionprefix } From 79038df4d3eae65cbfa51ed46a0e82692e4e75ec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 00:32:36 +0000 Subject: [PATCH 339/589] Add filtered publishing --- publish/do.groovy | 66 ++++++++++++++++++++++++++++++++++++++---- publish/publish.groovy | 2 +- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 7844289..7ec8be8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -5,6 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" tempfolder = "${WORKSPACE}/temp" def resetfolders() { @@ -14,6 +15,9 @@ def resetfolders() { dir(releaserepofolder) { deleteDir() } + dir(filteredreleaserepofolder) { + deleteDir() + } dir(tempfolder) { deleteDir() } @@ -59,7 +63,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } } -def publish(String customer) { +def publish(String customer, String finalreleaserepofolder = releaserepofolder) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' @@ -67,15 +71,41 @@ def publish(String customer) { def String FTP_USER='ftp@globalinkjetsystems.com' def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' - dir(releaserepofolder) { + dir(finalreleaserepofolder) { summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash - ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ summaryappend(", Transferred ") } } +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdir(); + } + dirFrom.eachFile(FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def filteredpublish(itempathstopublish, customer) { + for (String itempath : itempathstopublish) { + def srcpath = "${releaserepofolder}/${itempath}" + File srcpathfile = new File(srcpath) + def dstpath = "${filteredreleaserepofolder}/${itempath}" + File dstpathfile = new File(dstpath) + def dstpathdirname = dstpathfile.getParent() + copydirectory(srcpath, dstpath) + } + publish(customer, filteredreleaserepofolder) +} + def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" @@ -102,11 +132,25 @@ def stripgithistory() { } } -def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + } + return paths +} + +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() preconditions() - + // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) @@ -122,9 +166,19 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin recallcomponent(releaseinfo, softwarerelease) } stripgithistory() - publish(customer) + + itempathstopublish = convertitemstopublish(itemstopublishstring) + if (itempathstopublish.isEmpty()) { + publish(customer) + } else { + filteredpublish(itempathstopublish, customer) + } return [summarystring, markunstable, unstablereason] } +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(releasemeta, "", customer, softwarerelease, maxdepth) +} + return this; diff --git a/publish/publish.groovy b/publish/publish.groovy index a703c9e..b478a5d 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -44,7 +44,7 @@ pipeline { steps { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' From 9fad59a8a8c08b0fac03e5160d21212e0cdb9874 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 07:44:00 +0000 Subject: [PATCH 340/589] Add filtered publishing --- publish/do.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/do.groovy b/publish/do.groovy index 7ec8be8..996123c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -101,6 +101,7 @@ def filteredpublish(itempathstopublish, customer) { def dstpath = "${filteredreleaserepofolder}/${itempath}" File dstpathfile = new File(dstpath) def dstpathdirname = dstpathfile.getParent() + echo "Copying recursively from ${srcpath} to ${dstpath}" copydirectory(srcpath, dstpath) } publish(customer, filteredreleaserepofolder) @@ -140,6 +141,7 @@ def convertitemstopublish(String itemstopublishstring) { def String[] pathitems = item.split('#') def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" paths.add(path) + echo "Selected item to publish: ${path}" } return paths } From 9ee093481dc01085da4318e4ce3e12a32e28270a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:34:07 +0000 Subject: [PATCH 341/589] Add filtered publishing --- publish/do.groovy | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 996123c..572c2e3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -41,6 +41,16 @@ def summaryappend(String summary) { summarystring += summary } +def guessreporoot(componentURL) { + if (env.REPOROOT || env.REPOROOT.length() == 0) { + return '' + } + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + def reporoot = "${REPOROOT}/${guess}" + echo "Guessing reporoot is: ${reporoot}" + return reporoot +} + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder @@ -56,7 +66,8 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { publishablePaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone + def reporoot = guessreporoot(componentURL) + gh.gitCheckout(componentURL, revision, repofolder, reporoot, 1, true, publishablePaths) // Shallow clone } else { summaryappend("None ") From 6a334f56f12470738342f88bdb58c28d06c317db Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:42:18 +0000 Subject: [PATCH 342/589] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 572c2e3..ade1c3f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -42,7 +42,7 @@ def summaryappend(String summary) { } def guessreporoot(componentURL) { - if (env.REPOROOT || env.REPOROOT.length() == 0) { + if (!env.REPOROOT || env.REPOROOT.size() == 0) { return '' } def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') From e328e823e4db25723952a5a900c20c9bd7a9cbcd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:47:47 +0000 Subject: [PATCH 343/589] Add filtered publishing --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index ade1c3f..14286b9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -42,11 +42,11 @@ def summaryappend(String summary) { } def guessreporoot(componentURL) { - if (!env.REPOROOT || env.REPOROOT.size() == 0) { + if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { return '' } def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPOROOT}/${guess}" + def reporoot = "${REPO_ROOT}/${guess}" echo "Guessing reporoot is: ${reporoot}" return reporoot } From 94a966c7cd4e055acf894b34a4edd4bcdd3d4740 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:52:40 +0000 Subject: [PATCH 344/589] Add filtered publishing --- publish/do.groovy | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 14286b9..e637e8f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -41,14 +41,9 @@ def summaryappend(String summary) { summarystring += summary } -def guessreporoot(componentURL) { - if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { - return '' - } +def guessreporootsuffix(componentURL) { def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPO_ROOT}/${guess}" - echo "Guessing reporoot is: ${reporoot}" - return reporoot + return guess } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") @@ -66,8 +61,8 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { publishablePaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } - def reporoot = guessreporoot(componentURL) - gh.gitCheckout(componentURL, revision, repofolder, reporoot, 1, true, publishablePaths) // Shallow clone + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckout(componentURL, revision, reporootsuffix, reporoot, 1, true, publishablePaths) // Shallow clone } else { summaryappend("None ") From 944f732e896a578636bc72be9851eba8032aa0ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:54:45 +0000 Subject: [PATCH 345/589] Add filtered publishing --- versioncontrol/git.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f7d10bb..9a313f2 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,6 +19,16 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } +def guessreporoot(componentURL) { + if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { + return '' + } + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + def reporoot = "${REPO_ROOT}/${guess}" + echo "Guessing reporoot is: ${reporoot}" + return reporoot +} + def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRootSuffix != "") { From 7c4d1df78d2022f6ba5fcf626d45257d35fee93c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 08:56:32 +0000 Subject: [PATCH 346/589] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index e637e8f..20e332a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -62,7 +62,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { summaryappend("${publishableitem.filepath},") } def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckout(componentURL, revision, reporootsuffix, reporoot, 1, true, publishablePaths) // Shallow clone + gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablePaths) // Shallow clone } else { summaryappend("None ") From df095fec9622a63f8591a9a5d65d11979c6af685 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:04:52 +0000 Subject: [PATCH 347/589] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 20e332a..c3d229e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -108,7 +108,7 @@ def filteredpublish(itempathstopublish, customer) { File dstpathfile = new File(dstpath) def dstpathdirname = dstpathfile.getParent() echo "Copying recursively from ${srcpath} to ${dstpath}" - copydirectory(srcpath, dstpath) + copydirectory(srcpathfile, dstpathfile) } publish(customer, filteredreleaserepofolder) } From 889d2b87977862468d0358e752395773537e10e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:20:02 +0000 Subject: [PATCH 348/589] Add filtered publishing --- publish/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c3d229e..eea1015 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -88,7 +88,7 @@ def publish(String customer, String finalreleaserepofolder = releaserepofolder) def copydirectory(File dirFrom, File dirTo) { if (!dirTo.exists()) { - dirTo.mkdir(); + dirTo.mkdirs(); } dirFrom.eachFile(FILES) { File source -> File target = new File(dirTo, source.getName()) @@ -103,12 +103,12 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { def srcpath = "${releaserepofolder}/${itempath}" - File srcpathfile = new File(srcpath) + def File srcpathfile = new File(srcpath) def dstpath = "${filteredreleaserepofolder}/${itempath}" - File dstpathfile = new File(dstpath) - def dstpathdirname = dstpathfile.getParent() - echo "Copying recursively from ${srcpath} to ${dstpath}" - copydirectory(srcpathfile, dstpathfile) + def File dstpathfile = new File(dstpath) + def dstpathparentdir = dstpathfile.getParentFile() + echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getName()}" + copydirectory(srcpathfile, dstpathparentdir) } publish(customer, filteredreleaserepofolder) } From e4e83a02f747485eb5f2aa47e63aaa0c9757aea0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:26:40 +0000 Subject: [PATCH 349/589] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index eea1015..255fae9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -107,7 +107,7 @@ def filteredpublish(itempathstopublish, customer) { def dstpath = "${filteredreleaserepofolder}/${itempath}" def File dstpathfile = new File(dstpath) def dstpathparentdir = dstpathfile.getParentFile() - echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getName()}" + echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" copydirectory(srcpathfile, dstpathparentdir) } publish(customer, filteredreleaserepofolder) From 9160ff3e7c3600be0bda79ef55d94520151f5e11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:45:43 +0000 Subject: [PATCH 350/589] Add filtered publishing --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 255fae9..e5e7965 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,11 +90,11 @@ def copydirectory(File dirFrom, File dirTo) { if (!dirTo.exists()) { dirTo.mkdirs(); } - dirFrom.eachFile(FILES) { File source -> + dirFrom.eachFile(FileType.FILES) { File source -> File target = new File(dirTo, source.getName()) target.bytes = source.bytes } - dirFrom.eachFile(DIRECTORIES) { File source -> + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> File target = new File(dirTo, source.getName()) copyLargeDir(source, target) } From d868c21c174d8157ef3820dd0bd0ea1dc5699325 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:48:13 +0000 Subject: [PATCH 351/589] Add filtered publishing --- publish/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index e5e7965..7ce778c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -143,12 +143,14 @@ def convertitemstopublish(String itemstopublishstring) { def List paths = [] def String[] items = itemstopublishstring.split(',') + def String allpaths for(String item : items) { def String[] pathitems = item.split('#') def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" paths.add(path) - echo "Selected item to publish: ${path}" + allpaths += path } + echo "Selected item to publish: ${allpaths}" return paths } From 76f24173a3091d3c5b652159adb3769bfe917374 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 11:54:08 +0000 Subject: [PATCH 352/589] Add filtered publishing --- publish/do.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 7ce778c..c4bf7eb 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,3 +1,5 @@ +import groovy.io.FileType + markunstable = false unstablereason = "" @@ -143,14 +145,14 @@ def convertitemstopublish(String itemstopublishstring) { def List paths = [] def String[] items = itemstopublishstring.split(',') - def String allpaths + def String allpaths = "" for(String item : items) { def String[] pathitems = item.split('#') def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" paths.add(path) - allpaths += path + allpaths += "${path}\n" } - echo "Selected item to publish: ${allpaths}" + echo "Selected item to publish:\n${allpaths}" return paths } From 7f8b0cf8667f2df714f1aeefc187e0366b148d4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 12:54:50 +0000 Subject: [PATCH 353/589] Add filtered publishing --- publish/do.groovy | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c4bf7eb..749497a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,13 +104,16 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { - def srcpath = "${releaserepofolder}/${itempath}" - def File srcpathfile = new File(srcpath) - def dstpath = "${filteredreleaserepofolder}/${itempath}" - def File dstpathfile = new File(dstpath) - def dstpathparentdir = dstpathfile.getParentFile() - echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" - copydirectory(srcpathfile, dstpathparentdir) + def srcpathglob = "${releaserepofolder}/${itempath}" + def srcpaths = new FileNameFinder().getFileNames(srcpathglob) + for (String srcpath: srcpaths) { + def File srcpathfile = new File(srcpath) + def dstpath = "${filteredreleaserepofolder}/${itempath}" + def File dstpathfile = new File(dstpath) + def dstpathparentdir = dstpathfile.getParentFile() + echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" + copydirectory(srcpathfile, dstpathparentdir) + } } publish(customer, filteredreleaserepofolder) } From b822e39e2a591c86fe35ad4ed014fff8a7d5205a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:04:43 +0000 Subject: [PATCH 354/589] Add filtered publishing --- publish/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 749497a..7262a3a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,8 +104,8 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { - def srcpathglob = "${releaserepofolder}/${itempath}" - def srcpaths = new FileNameFinder().getFileNames(srcpathglob) + // Convert glob wildcard paths to real paths + def srcpaths = new FileNameFinder().getFileNames(releaserepofolder, itempath) for (String srcpath: srcpaths) { def File srcpathfile = new File(srcpath) def dstpath = "${filteredreleaserepofolder}/${itempath}" From f9a078c895c5facd1c3f8273d416907444169abe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:15:54 +0000 Subject: [PATCH 355/589] Add filtered publishing --- publish/do.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/do.groovy b/publish/do.groovy index 7262a3a..2d5ee05 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -105,7 +105,9 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { // Convert glob wildcard paths to real paths + echo "Finding files at base folder: ${releaserepofolder} and glob filter: ${itempath}" def srcpaths = new FileNameFinder().getFileNames(releaserepofolder, itempath) + echo "Paths found: ${srcpaths}" for (String srcpath: srcpaths) { def File srcpathfile = new File(srcpath) def dstpath = "${filteredreleaserepofolder}/${itempath}" From 36ca33077f74c74c11dc747d33e345a165a904ee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:34:40 +0000 Subject: [PATCH 356/589] Add filtered publishing --- publish/do.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 2d5ee05..732373e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,9 +104,14 @@ def copydirectory(File dirFrom, File dirTo) { def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { - // Convert glob wildcard paths to real paths - echo "Finding files at base folder: ${releaserepofolder} and glob filter: ${itempath}" - def srcpaths = new FileNameFinder().getFileNames(releaserepofolder, itempath) + def defaultsrcpath = "${releaserepofolder}/${itempath}" + def srcpaths = [ defaultsrcpath ] + if (itempath.endsWith('**') { // Convert glob wildcard paths to real paths + File srcfilepath = new File(defaultsrcpath).getParentFile() + def globpattern = '**' + echo "Finding files at base folder: ${srcfilepath} and glob filter: ${globpattern}" + srcpaths = new FileNameFinder().getFileNames(releaserepofolder, globpattern) + } echo "Paths found: ${srcpaths}" for (String srcpath: srcpaths) { def File srcpathfile = new File(srcpath) From 36ea650ea7dfa7601b75ae3244bfb0a80f3390f7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 13:42:43 +0000 Subject: [PATCH 357/589] Add filtered publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 732373e..dcc88ca 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -106,7 +106,7 @@ def filteredpublish(itempathstopublish, customer) { for (String itempath : itempathstopublish) { def defaultsrcpath = "${releaserepofolder}/${itempath}" def srcpaths = [ defaultsrcpath ] - if (itempath.endsWith('**') { // Convert glob wildcard paths to real paths + if (itempath.endsWith('**')) { // Convert glob wildcard paths to real paths File srcfilepath = new File(defaultsrcpath).getParentFile() def globpattern = '**' echo "Finding files at base folder: ${srcfilepath} and glob filter: ${globpattern}" From aa100d8731cea24e75714c07a85e86c011c4040b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:21:51 +0000 Subject: [PATCH 358/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 68 ++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index dcc88ca..a10fe51 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -25,6 +25,14 @@ def resetfolders() { } } +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + def preconditions() { markunstable = false unstablereason = "" @@ -49,7 +57,7 @@ def guessreporootsuffix(componentURL) { } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -57,14 +65,18 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo - def ArrayList publishablePaths = new ArrayList() + def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + if (itempathstopublish.size() == 0 || publishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } } + } + if (publishablepaths.size() > 0) { def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablePaths) // Shallow clone + gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone } else { summaryappend("None ") @@ -102,38 +114,22 @@ def copydirectory(File dirFrom, File dirTo) { } } -def filteredpublish(itempathstopublish, customer) { - for (String itempath : itempathstopublish) { - def defaultsrcpath = "${releaserepofolder}/${itempath}" - def srcpaths = [ defaultsrcpath ] - if (itempath.endsWith('**')) { // Convert glob wildcard paths to real paths - File srcfilepath = new File(defaultsrcpath).getParentFile() - def globpattern = '**' - echo "Finding files at base folder: ${srcfilepath} and glob filter: ${globpattern}" - srcpaths = new FileNameFinder().getFileNames(releaserepofolder, globpattern) - } - echo "Paths found: ${srcpaths}" - for (String srcpath: srcpaths) { - def File srcpathfile = new File(srcpath) - def dstpath = "${filteredreleaserepofolder}/${itempath}" - def File dstpathfile = new File(dstpath) - def dstpathparentdir = dstpathfile.getParentFile() - echo "Copying recursively from ${srcpath} to ${dstpathparentdir.getAbsolutePath()}" - copydirectory(srcpathfile, dstpathparentdir) - } - } - publish(customer, filteredreleaserepofolder) +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" } -def recallcomponent(releaseinfo, String softwarerelease) { +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + +def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) + def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) summaryappend("\n") } @@ -183,19 +179,13 @@ def publishhierarchy(releasemeta, String itemstopublishstring, String customer, } echo versionuniquecomponent + def itempathstopublish = convertitemstopublish(itemstopublishstring) for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] - recallcomponent(releaseinfo, softwarerelease) + recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } stripgithistory() - itempathstopublish = convertitemstopublish(itemstopublishstring) - if (itempathstopublish.isEmpty()) { - publish(customer) - } else { - filteredpublish(itempathstopublish, customer) - } - return [summarystring, markunstable, unstablereason] } From e05f6a2da23d5c6d621365322c3fb3200edf3feb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:26:14 +0000 Subject: [PATCH 359/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 8 -------- 1 file changed, 8 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index a10fe51..62dea7c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -114,14 +114,6 @@ def copydirectory(File dirFrom, File dirTo) { } } -def uniquereleasefolder(String softwarerelease, releasemeta) { - return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" -} - -def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { - return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" -} - def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) From 755172bc503eb7e7bd9b89c7c67c282b9815e1c2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:32:52 +0000 Subject: [PATCH 360/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/publish/do.groovy b/publish/do.groovy index 62dea7c..53fc791 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -65,6 +65,10 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo + + echo "itempathstopublish: ${itempathstopublish}" + echo "publishableitems: ${publishableitems}" + def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { From 14a6291d3dd39cbddfad5c92cc2715a7c105c4cc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:40:16 +0000 Subject: [PATCH 361/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 53fc791..f4aad08 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -66,13 +66,10 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo - echo "itempathstopublish: ${itempathstopublish}" - echo "publishableitems: ${publishableitems}" - def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - if (itempathstopublish.size() == 0 || publishableitem in itempathstopublish) { + if (itempathstopublish.size() == 0 || publishableitem.filepath in itempathstopublish) { publishablepaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } From 19932d9f1549f8d9cfc449f07e9d4127d94b8492 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:50:46 +0000 Subject: [PATCH 362/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/do.groovy b/publish/do.groovy index f4aad08..9073b85 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -69,6 +69,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { + echo "publishableitem: ${publishableitem.filepath}" if (itempathstopublish.size() == 0 || publishableitem.filepath in itempathstopublish) { publishablepaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") From d112e4f7d187ee601f6ecd1d9c1ba1b12808dafc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 14:59:06 +0000 Subject: [PATCH 363/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 9073b85..80113f3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -25,8 +25,12 @@ def resetfolders() { } } +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + def uniquereleasefolder(String softwarerelease, releasemeta) { - return "Software/${PRODUCT_NAME} ${softwarerelease}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + return "Software/${PRODUCT_NAME} ${softwarerelease}/${productuniquefolder(releasemeta)}" } def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { @@ -69,8 +73,9 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - echo "publishableitem: ${publishableitem.filepath}" - if (itempathstopublish.size() == 0 || publishableitem.filepath in itempathstopublish) { + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { publishablepaths.add(publishableitem.filepath) summaryappend("${publishableitem.filepath},") } From 5cac7a574bdba3dcf6dd31516ff249d7813eebb1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Mar 2019 15:53:27 +0000 Subject: [PATCH 364/589] Add filtered publishing - different approach for filtering --- publish/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/do.groovy b/publish/do.groovy index 80113f3..1a54b23 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -184,6 +184,7 @@ def publishhierarchy(releasemeta, String itemstopublishstring, String customer, recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } stripgithistory() + publish(customer) return [summarystring, markunstable, unstablereason] } From fa989ef12f6ac06ac840cc1b562e9590fd242883 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 22 Mar 2019 13:36:45 +0000 Subject: [PATCH 365/589] SMB-119 - Not barfing when no substitions occurred --- versionstamping/versionstamper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 3e59c6f..4c380a6 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -26,15 +26,18 @@ def main(argv): print("Could not find file: {0}\n".format(filetoconvert)) return 1 + count = 0 with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: fileread = c.read() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) - #print ("DEBUG: %s "%converted) - print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) f.write(converted) + if count == 0: + return 1 return 0 if __name__ == "__main__": From d1741859bc1a0c42885e702edd9b9fce8dd7d31a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:37:17 +0000 Subject: [PATCH 366/589] Add some checking job scripts --- check/build_meta.groovy | 86 +++++++++++++++++++++++++++++++++++++++++ check/do_meta.groovy | 64 ++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 check/build_meta.groovy create mode 100644 check/do_meta.groovy diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..34eed98 --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,86 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..75353da --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,64 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + def offsetpath = buildinfo.offsetpath + versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; From d75c59e92ada16a48032b23a85d5d5dc89ea097d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:44:08 +0000 Subject: [PATCH 367/589] Add some checking job scripts --- check/do_meta.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 75353da..6712a11 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,14 +47,15 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() - // Walk the meta information to a max depth, looking for components to release + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath - versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 86fc6f3b5eb22b2dd35a792fce3acfc485a045ec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:07:24 +0000 Subject: [PATCH 368/589] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 6712a11..e01aa41 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 64fecfb6b3208ae982a277d34d0bcfe3dae893a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:12:31 +0000 Subject: [PATCH 369/589] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index e01aa41..7fce1eb 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 741a9f966c0822e4b678165cab07c6f1dbfb3fe0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:22:58 +0000 Subject: [PATCH 370/589] Add some checking job scripts --- check/do_meta.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 7fce1eb..cb19ddf 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,6 +47,18 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = buildinfo.meta.componentversionuniqueidentifier() + def offsetpath = buildinfo.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" From 401ce43ea3d42a469fa41d588ab5e3a464059b0e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:25:25 +0000 Subject: [PATCH 371/589] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index cb19ddf..8935ca5 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -64,8 +64,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - def offsetpath = buildinfo.offsetpath + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.offsetpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } From c59af13d9c6c6668d7486eb391c1daa4b8dc81ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:29:23 +0000 Subject: [PATCH 372/589] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 8935ca5..7bfe2d1 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = buildinfo.meta.componentversionuniqueidentifier() - def offsetpath = buildinfo.offsetpath + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } From 6555898bb55b91d61dae523daa9af06c6d6e3411 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:51:03 +0000 Subject: [PATCH 373/589] Add some checking job scripts --- meta/folder.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d751f5e..a18e8d7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,6 +687,18 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } + Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + } + } else { + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetacollection + } + Map releasewalkerunique(int maxdepth, String startpath = "") { def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) @@ -694,7 +706,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) } return releasemetauniquecollection @@ -707,7 +719,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetaversionuniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) } return releasemetaversionuniquecollection From 1ed3b8214c7e945d3ebe452561e2131cc0d99e34 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:54:50 +0000 Subject: [PATCH 374/589] Add some checking job scripts --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a18e8d7..a8ffa41 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,14 +687,14 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } - Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } } else { - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } return releasemetacollection } From 323da9fdf3edf65330275c026cc8828858b56fdb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:57:14 +0000 Subject: [PATCH 375/589] Add some checking job scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8ffa41..1bf855c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -690,7 +690,7 @@ class MetaInfo extends GISJsonSerialised { Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { releasemetacollection.put(uniquestring, relatedmetainfo) } } else { From 9e0f24f0d26d9c2f511d9cdbc32f6bfb6d4fd0f3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 12:21:15 +0000 Subject: [PATCH 376/589] Create a script to copy logic to a build job - so that it will be stamped during release --- copy/logic.groovy | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 copy/logic.groovy diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..6f8f633 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,56 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/logic", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying logic artifacts') { + steps { + script { + env.LOGICDEST = "${WORKSPACE}/logic" + if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["logic/**"]) + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} From 88e955e7653d8b2dc4a42a333b28a385911f191d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Mar 2019 14:21:19 +0000 Subject: [PATCH 377/589] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6f8f633..d951ffa 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -35,6 +35,9 @@ pipeline { script { def artifactlist = meta.generateartifacts(["logic/**"]) archiveArtifacts "${artifactlist}" + sh """ + cp -v -R logic/meta/info.json meta/info.json + """ } } } From 3c0b2178a810d4648220e5b258719a8d8a05ec3d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Mar 2019 14:25:25 +0000 Subject: [PATCH 378/589] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index d951ffa..845f676 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -34,10 +34,10 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["logic/**"]) - archiveArtifacts "${artifactlist}" sh """ cp -v -R logic/meta/info.json meta/info.json """ + archiveArtifacts "${artifactlist}" } } } From 6a600862582d3b19a59e358007c038d471c3bc05 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 08:56:29 +0000 Subject: [PATCH 379/589] Handle the artifact source parameter --- import/importer.groovy | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/import/importer.groovy b/import/importer.groovy index 5c996b4..40151e1 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -35,4 +35,18 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + switch (source) { + case "Jenkins job artifact": { + return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + case "Release repository": { + return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + return this; From 11b7fe881da3416d70bde83001592cc56f7aa9d8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 08:59:37 +0000 Subject: [PATCH 380/589] Handle the artifact source parameter --- import/importer.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 40151e1..abf3b2f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -37,12 +37,10 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { - case "Jenkins job artifact": { + case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) - } - case "Release repository": { + case "Release repository": return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) - } default: error("Unknown type of artifact source: ${source}") } From 05afef18481f42696eded8a57404336f63cfd98e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 09:16:18 +0000 Subject: [PATCH 381/589] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 845f676..fae69b4 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -13,6 +13,7 @@ pipeline { script { cleanfolders = [ "${WORKSPACE}/logic", + "${WORKSPACE}/embedded_logic", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -33,9 +34,10 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["logic/**"]) + def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - cp -v -R logic/meta/info.json meta/info.json + rm -v meta/info.json + mv -v logic/embedded_logic . """ archiveArtifacts "${artifactlist}" } From ff733458f94621a24c3af027bd2ae73db169561d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 09:19:44 +0000 Subject: [PATCH 382/589] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index fae69b4..6e97510 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,8 +36,8 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v meta/info.json - mv -v logic/embedded_logic . + rm -v -rf meta + mv -v logic/* . """ archiveArtifacts "${artifactlist}" } From ec33ceb812e97971316553e88676b32304aa8162 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 09:06:33 +0000 Subject: [PATCH 383/589] Import artifacts from repo --- import/importer.groovy | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index abf3b2f..e605743 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,4 @@ -versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( @@ -35,6 +35,18 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + gh.gitCheckout(rooturl, rootrevision, destpath) + + def importedmeta = meta.importrelated(destpath) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": From 51ba9bc94e85d07a464621c1d7d1c8d15f5ec673 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 09:25:47 +0000 Subject: [PATCH 384/589] Import artifacts from repo --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index e605743..fc8395f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,5 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( From 3745e27b9cb5d199044a693bf01f25e686257b7f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 13:57:21 +0000 Subject: [PATCH 385/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 9 +++++++++ meta/folder.groovy | 10 +++++++++- release/release.groovy | 3 +-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index fc8395f..133edba 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,6 +42,15 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.gitCheckout(rooturl, rootrevision, destpath) def importedmeta = meta.importrelated(destpath) + + for(associatedmeta in getassociatedmeta) { + def cloneurl = associatedmeta.releaseinfo.releaserepoURL + def clonerevision = associatedmeta.releasetag() + def clonepath = "${destpath}/${associatedmeta.offsetpath}" + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckout(cloneurl, rootrevision, clonepath) + } + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } diff --git a/meta/folder.groovy b/meta/folder.groovy index 1bf855c..053c38e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -477,6 +477,10 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } + ArrayList getassociatedmeta() { + return associatedmeta + } + List configurationinfoserialised() { def configuration = [] for (configurationitem in configurationinfo) { @@ -684,7 +688,11 @@ class MetaInfo extends GISJsonSerialised { } String componentversionuniqueidentifier() { - return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp + return componentidentifier() + "_" + buildstampinfo.buildstamp + } + + String releasetag() { + return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber } Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { diff --git a/release/release.groovy b/release/release.groovy index b0530be..3b5321d 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,8 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" - OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" + OTHER_RELEASE_TAGS = "${buildmeta.releasetag()},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason From 6056ba5874063695b74e72cee121b961e1cb3173 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 14:03:22 +0000 Subject: [PATCH 386/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 133edba..09451ec 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,7 +43,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - for(associatedmeta in getassociatedmeta) { + for(associatedmeta in importedmeta.getassociatedmeta()) { def cloneurl = associatedmeta.releaseinfo.releaserepoURL def clonerevision = associatedmeta.releasetag() def clonepath = "${destpath}/${associatedmeta.offsetpath}" From 659859878d3aad5e532b7c2136d0c2b328030d40 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 14:19:39 +0000 Subject: [PATCH 387/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 09451ec..2392b83 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,10 +43,12 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - for(associatedmeta in importedmeta.getassociatedmeta()) { - def cloneurl = associatedmeta.releaseinfo.releaserepoURL - def clonerevision = associatedmeta.releasetag() - def clonepath = "${destpath}/${associatedmeta.offsetpath}" + for(relatedmetainfo in importedmeta.getassociatedmeta()) { + def relatedmeta = relatedmetainfo.meta + def relatedoffsetpath = relatedmetainfo.offsetpath + def cloneurl = relatedmeta.releaseinfo.releaserepoURL + def clonerevision = "refs/tags/${relatedmeta.releasetag()}" + def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" gh.gitCheckout(cloneurl, rootrevision, clonepath) } From 9160063d947034622003e95e24eb2e31090e413e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 14:31:49 +0000 Subject: [PATCH 388/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 2392b83..c554173 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,7 +43,8 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - for(relatedmetainfo in importedmeta.getassociatedmeta()) { + relatedimportinfo = importedmeta.getassociatedmeta() + for(relatedmetainfo in relatedimportinfo) { def relatedmeta = relatedmetainfo.meta def relatedoffsetpath = relatedmetainfo.offsetpath def cloneurl = relatedmeta.releaseinfo.releaserepoURL From ab18f25218f5afe0402fbc3e9f017ebdbf4af0a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 16:45:17 +0000 Subject: [PATCH 389/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 3 ++- meta/folder.groovy | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index c554173..cdc2e1a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -43,7 +43,8 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def importedmeta = meta.importrelated(destpath) - relatedimportinfo = importedmeta.getassociatedmeta() + relatedimportinfo = importedmeta.getassociated() + echo "Cloning associated repos" for(relatedmetainfo in relatedimportinfo) { def relatedmeta = relatedmetainfo.meta def relatedoffsetpath = relatedmetainfo.offsetpath diff --git a/meta/folder.groovy b/meta/folder.groovy index 053c38e..987041b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -477,7 +477,7 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } - ArrayList getassociatedmeta() { + List getassociated() { return associatedmeta } @@ -635,12 +635,11 @@ class MetaInfo extends GISJsonSerialised { ] return data } - static String folder() { return METAFOLDERNAME } - + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" From 5006d6cd184fc11e30f1c99edf4dc390cb0e6d9a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 16:48:58 +0000 Subject: [PATCH 390/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index cdc2e1a..78f89cc 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -39,6 +39,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" gh.gitCheckout(rooturl, rootrevision, destpath) def importedmeta = meta.importrelated(destpath) From e23d564cb589081f986f9181d48e0a3e0b9fd701 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 16:57:54 +0000 Subject: [PATCH 391/589] When importing artifacts from repo, clone subcomponents --- meta/folder.groovy | 6 +++++- release/release.groovy | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 987041b..e805118 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -690,10 +690,14 @@ class MetaInfo extends GISJsonSerialised { return componentidentifier() + "_" + buildstampinfo.buildstamp } - String releasetag() { + String globalreleasetag() { return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber } + String releasetag() { + return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber + } + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] diff --git a/release/release.groovy b/release/release.groovy index 3b5321d..79c0234 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,7 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - OTHER_RELEASE_TAGS = "${buildmeta.releasetag()},${OTHER_RELEASE_TAGS}" + OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason From 7dd532095cc0e8d597fb4cf41c9b1464b0708ca1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 17:00:25 +0000 Subject: [PATCH 392/589] When importing artifacts from repo, clone subcomponents --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 78f89cc..c6b80a4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -53,7 +53,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckout(cloneurl, rootrevision, clonepath) + gh.gitCheckout(cloneurl, clonerevision, clonepath) } if (incrementversionnumber) { From 8cd99c0927f3a215c996e88c8896cb5a5c12077e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:33:55 +0000 Subject: [PATCH 393/589] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- import/importer.groovy | 16 +++++++++++++++- meta/folder.groovy | 10 +++++++++- publish/do.groovy | 15 +-------------- versioncontrol/git.groovy | 13 +++++++++++++ 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index c6b80a4..c945ec0 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -36,6 +36,12 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def rewritemetafileasreleased(meta, String filepath) { + // Modify the meta info to say that is has already been released + meta.releaseinfo.markreleased() + meta.saveJsonSerialised(filepath) +} + def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() @@ -54,8 +60,16 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" gh.gitCheckout(cloneurl, clonerevision, clonepath) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) } - + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${relatedmeta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } diff --git a/meta/folder.groovy b/meta/folder.groovy index e805118..ec011ae 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -219,11 +219,13 @@ class BuildStampInfo extends GISJsonSerialised { class ReleaseInfo extends GISJsonSerialised { String releaserepoURL ArrayList releaseitems + boolean released - ReleaseInfo(script, releaserepoURL = "") { + ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { super(script) this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() + this.released = released } void set(String releaserepoURL = "") { @@ -234,6 +236,10 @@ class ReleaseInfo extends GISJsonSerialised { releaseitems.add(releaseitem) } + void markreleased() { + this.released = true + } + List releaseitemssserialised() { def items = [] for (ri in releaseitems) { @@ -248,12 +254,14 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } + released ?= rawmeta['released'] : false } LinkedHashMap serialisationdata() { def data = [ 'releaserepoURL' : releaserepoURL, 'releaseitems' : releaseitemssserialised(), + 'released' : released ] return data } diff --git a/publish/do.groovy b/publish/do.groovy index 1a54b23..b63b15c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -133,19 +133,6 @@ def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { summaryappend("\n") } -def stripgithistory() { - echo "Stripping git history out of ${releaserepofolder}" - dir(releaserepofolder) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rvf XX - find . -name ".gitignore" | xargs -iXX rm -rvf XX - find . -name ".gitmodules" | xargs -iXX rm -rvf XX - find . -iname ".gitattributes" | xargs -iXX rm -rvf XX - """ - } -} - def convertitemstopublish(String itemstopublishstring) { def List paths = [] @@ -183,7 +170,7 @@ def publishhierarchy(releasemeta, String itemstopublishstring, String customer, def releaseinfo = componentReleaseInfoVersionUnique[key] recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } - stripgithistory() + gh.stripgithistory(releaserepofolder) publish(customer) return [summarystring, markunstable, unstablereason] diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 9a313f2..7a1dd17 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -89,4 +89,17 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { return true } +def stripgithistory(String rootfolder) { + echo "Stripping git history out of ${rootfolder}" + dir(rootfolder) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX + """ + } +} + return this From 5659c8eeb41dbc2dcd48b1a7659ac1581e937287 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:36:35 +0000 Subject: [PATCH 394/589] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ec011ae..be85584 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -254,7 +254,7 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } - released ?= rawmeta['released'] : false + released = rawmeta['released'] ?= false } LinkedHashMap serialisationdata() { From a5dc59e6b297e8728cb9bcb5723f65f9427318c1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:38:02 +0000 Subject: [PATCH 395/589] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index be85584..a868855 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -254,7 +254,7 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } - released = rawmeta['released'] ?= false + released = rawmeta['released'] ?: false } LinkedHashMap serialisationdata() { From 49783e0f468deb3318ce65a0024df3a32c200984 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 18:43:58 +0000 Subject: [PATCH 396/589] When importing artifacts from repo, clone subcomponents; Remove git history; Mark them as released --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index c945ec0..a23ee83 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -66,7 +66,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, rewritemetafileasreleased(relatedmeta, metafilepath) } gh.stripgithistory(destpath) - def metafilepath = "${destpath}/${relatedmeta.metafilepath()}" + def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(meta, metafilepath) From 86613fcb2cf9cab5dc11ec847a65dcfef9f67195 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 19:43:15 +0000 Subject: [PATCH 397/589] When releasing, take into account already-released components --- release/do.groovy | 61 ++++++++++++++++++++++++--------------- versioncontrol/git.groovy | 8 ++--- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 604581a..fd2b06b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -172,22 +172,29 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele return taglist } -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false +def String releasegitgetshacode(String tag) { + def existingshacode = "" // Check if tag exists already def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() + existingshacode = readFile(file: gitrevparseoutputfile).trim() + } + return existingshacode +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def String existingshacode = releasegitgetshacode(tag) + if (existingshacode) { // Tag exists if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") summaryappend("(${tag} - Could not add!)") - } - else { + } else { summaryappend("(${tag} - Already added)") } - } - else { // If tag does not exist already + } else { // If tag does not exist already // Tag sh("git tag \"${tag}\"") summaryappend("(${tag})") @@ -270,26 +277,35 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp - def String buildstamptag = "${buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" + def String buildstamptag = buildmeta.releasetag() dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - releasegitadd() + def boolean alreadyreleased = buildmeta.releaseinfo.released + echo "Already released: ${alreadyreleased}" - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamptag) - } - else { - releasegitreset() - summaryappend("No-commit,") - } + def String postcommitshacode + def boolean changed = false + if (alreadyreleased) { + summaryappend("Already-released,") + postcommitshacode = releasegitgetshacode(buildstamptag) + echo "Previous release sha code: ${postcommitshacode}" + } else { + releasegitadd() - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" + changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamptag) + } else { + releasegitreset() + summaryappend("No-commit,") + } + + postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + } ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) @@ -299,7 +315,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamptag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamptag}" + def String srcbuildstamptag = buildmeta.globalreleasetag() def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { @@ -308,8 +324,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source if (tagadded) { sourcegitpush(dryrunpush) } - } - else { + } else { summaryappend(",No Src ") } } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 7a1dd17..ba4d96b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -94,10 +94,10 @@ def stripgithistory(String rootfolder) { dir(rootfolder) { sh """#!/bin/bash pwd - find . -type d -name ".git" | xargs -iXX rm -rvf XX - find . -name ".gitignore" | xargs -iXX rm -rvf XX - find . -name ".gitmodules" | xargs -iXX rm -rvf XX - find . -iname ".gitattributes" | xargs -iXX rm -rvf XX + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX """ } } From 6eb0f77114338dd00296ebfe7a8837c62dec34a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Mar 2019 19:58:37 +0000 Subject: [PATCH 398/589] Recursively mark all associated items of meta as released --- import/importer.groovy | 2 +- meta/folder.groovy | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index a23ee83..34707bc 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -38,7 +38,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def rewritemetafileasreleased(meta, String filepath) { // Modify the meta info to say that is has already been released - meta.releaseinfo.markreleased() + meta.markreleased() meta.saveJsonSerialised(filepath) } diff --git a/meta/folder.groovy b/meta/folder.groovy index a868855..346b2ee 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -489,6 +489,13 @@ class MetaInfo extends GISJsonSerialised { return associatedmeta } + void markreleased() { + releaseinfo.markreleased() + for (associated in associatedmeta) { + associated.meta.releaseinfo.markreleased() + } + } + List configurationinfoserialised() { def configuration = [] for (configurationitem in configurationinfo) { From a756df369aae75957a999570992da95b59335fd1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:44:12 +0000 Subject: [PATCH 399/589] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ba4d96b..a474b6e 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -29,15 +29,50 @@ def guessreporoot(componentURL) { return reporoot } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRootSuffix != "") { - reporootpath = "${reporootenv.trim()}/${repoRootSuffix}" - echo "Using release repo: ${reporootpath}" - } else { - reporootpath = "" +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" - } + } else { + def reporootname = uniquereporootname(gitURL) + def reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootname}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootname}" + } + return reporootsubfolder +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', userRemoteConfigs: [ From df087201a3f9bb54c2b65561f4c95fc21be1c344 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:48:35 +0000 Subject: [PATCH 400/589] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a474b6e..bd1b15e 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,16 +19,6 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def guessreporoot(componentURL) { - if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { - return '' - } - def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPO_ROOT}/${guess}" - echo "Guessing reporoot is: ${reporoot}" - return reporoot -} - def String uniquereporootname(String gitURL) { def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" @@ -68,6 +58,7 @@ def String checkouttoreporoot(String gitURL) { } else { echo "Found ${gitURL} checked out at ${reporootname}" } + } return reporootsubfolder } From 9dd09e4444eabe13100dec7a3dbca9b5825b5f20 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:49:54 +0000 Subject: [PATCH 401/589] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index bd1b15e..ea3501c 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,7 +34,7 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - def reporootsubfolder = "${reporootenv}/${reporootname}" + reporootsubfolder = "${reporootenv}/${reporootname}" if (!fileExists(reporootsubfolder)) { echo "Checking out ${gitURL} to ${reporootname}" From d1e8531b5ad42b0997fc53036a555e9c25c961fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:52:55 +0000 Subject: [PATCH 402/589] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ea3501c..432316f 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,10 +34,10 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootname}" + reporootsubfolder = "${reporootenv}/${reporootsubfolder}" if (!fileExists(reporootsubfolder)) { - echo "Checking out ${gitURL} to ${reporootname}" + echo "Checking out ${gitURL} to ${reporootsubfolder}" checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -56,7 +56,7 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootname}" + echo "Found ${gitURL} checked out at ${reporootsubfolder}" } } return reporootsubfolder From add368c009a6f99a15c7bf6d2de25af96be5600d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:56:01 +0000 Subject: [PATCH 403/589] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 432316f..5599dc6 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,7 +34,7 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootsubfolder}" + reporootsubfolder = "${reporootenv}/${reporootname}" if (!fileExists(reporootsubfolder)) { echo "Checking out ${gitURL} to ${reporootsubfolder}" From 34a79713cdf76c175386f954080896955f6981f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Apr 2019 09:01:42 +0100 Subject: [PATCH 404/589] Check contents of reporoot repo --- versioncontrol/git.groovy | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 5599dc6..253ffee 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -56,7 +56,20 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootsubfolder}" + echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status" + dir(reporootsubfolder) { + if (isUnix()) { + sh """ + git status + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + bat """ + git status + if errorlevel 1 EXIT /b 1 + """ + } + } } } return reporootsubfolder From 67785ac859fb5e376be83a6e63e1d4b22007f410 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Apr 2019 17:14:15 +0100 Subject: [PATCH 405/589] Use the jenkins pipeline checkout mechanism for getting release and source repo --- release/do.groovy | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index fd2b06b..ada51bb 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -10,6 +10,7 @@ tempfolder = "${WORKSPACE}/temp" gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" metascript = autobuilder.getmetafolderscript() +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -57,7 +58,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - sh("git clone ${releaserepo} ${checkoutfolder}") + gh.gitCheckout(releaserepo, '*/master', checkoutfolder, "", 1, false) } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { @@ -69,27 +70,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl - if (sourcerepo) { - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } - } + gh.gitCheckout(sourcerepo, '*/master', checkoutfolder, "", 1, false) } def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { From fb7953a95909ad06a75d2f4e890a866811963482 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Apr 2019 17:24:13 +0100 Subject: [PATCH 406/589] Revert "Use the jenkins pipeline checkout mechanism for getting release and source repo" This reverts commit 6808492ff5bc48c3e5d6a3e4239c0c46b9d0145e. --- release/do.groovy | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index ada51bb..fd2b06b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -10,7 +10,6 @@ tempfolder = "${WORKSPACE}/temp" gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" metascript = autobuilder.getmetafolderscript() -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -58,7 +57,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - gh.gitCheckout(releaserepo, '*/master', checkoutfolder, "", 1, false) + sh("git clone ${releaserepo} ${checkoutfolder}") } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { @@ -70,7 +69,27 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl - gh.gitCheckout(sourcerepo, '*/master', checkoutfolder, "", 1, false) + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } } def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { From c9e2557aa9c7b48a11602393b53842ba5e23cfe3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 15:53:44 +0100 Subject: [PATCH 407/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 45 +++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 819a580..d064161 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -30,4 +30,47 @@ def placeBinaries(artifactDetails, installerDetails) { } } -return this; \ No newline at end of file +def checkISMfile(String ISMfile) { + def xmlFile = new File(ISMfile) + + def parsedxml = new XmlParser().parse(xmlFile) + def pathvariableinfo = [:] + def exitcode = 0 + // Load up a map of path variable and path pairs + // Ignore blank path definitions + // Ignore the workspace definition which should have the only absolute path + parsedxml.each{ table -> + if (table.@name == "ISPathVariable") { + table.row.each{ row -> + def pathname = row.td[0].value()[0] + def path = row.td[1].value()[0] + if (path && pathname != "WORKSPACE") { + pathvariableinfo[pathname] = path + } + } + } + } + // Check that each path variable is defined beginning with a path variable or the WORKSPACE one + pathvariableinfo.each{ pair -> + println "pair key: ${pair.key} value: ${pair.value}" + def path = pair.value + def pathvarmatcher = path =~ /<(.*)>.*/ + if (pathvarmatcher.matches()) { + def pathvarfound = pathvarmatcher.group(1) + if (pathvarfound != 'WORKSPACE') { + if (!pathvariableinfo.containsKey(pathvarfound)) { + println "Undefined path variable ${pathvarfound} found" + exitcode = 1 + } + } + } else { + println "Path: ${path} does not begin with a path variable" + exitcode = 1 + } + } + if (exitcode > 0) { + error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" + } +} + +return this; From eee2111033b7dd9ae16c553277a7f069f7782e3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 17:58:05 +0100 Subject: [PATCH 408/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index d064161..886c604 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -31,9 +31,9 @@ def placeBinaries(artifactDetails, installerDetails) { } def checkISMfile(String ISMfile) { - def xmlFile = new File(ISMfile) + def xml = readFile(ISMfile) - def parsedxml = new XmlParser().parse(xmlFile) + def parsedxml = new XmlParser().parseText(xml) def pathvariableinfo = [:] def exitcode = 0 // Load up a map of path variable and path pairs From fb6c1a0e31c5f112a449ad5f4c46bdc1e8d0dd46 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:07:31 +0100 Subject: [PATCH 409/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 886c604..6cecb85 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -33,7 +33,7 @@ def placeBinaries(artifactDetails, installerDetails) { def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) - def parsedxml = new XmlParser().parseText(xml) + def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] def exitcode = 0 // Load up a map of path variable and path pairs From 1fce006fe33c5932874df279d98bc391fb85ea2b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:21:53 +0100 Subject: [PATCH 410/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6cecb85..f3dcdd4 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -30,6 +30,7 @@ def placeBinaries(artifactDetails, installerDetails) { } } +@NonCPS def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) From 865deaecf843177e6f223ef3c676dcbbc1d96601 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:30:16 +0100 Subject: [PATCH 411/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index f3dcdd4..823c117 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -33,7 +33,8 @@ def placeBinaries(artifactDetails, installerDetails) { @NonCPS def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) - + echo "Checking ISM file: ${ISMfile}" + def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] def exitcode = 0 @@ -53,24 +54,26 @@ def checkISMfile(String ISMfile) { } // Check that each path variable is defined beginning with a path variable or the WORKSPACE one pathvariableinfo.each{ pair -> - println "pair key: ${pair.key} value: ${pair.value}" + echo "pair key: ${pair.key} value: ${pair.value}" def path = pair.value def pathvarmatcher = path =~ /<(.*)>.*/ if (pathvarmatcher.matches()) { def pathvarfound = pathvarmatcher.group(1) if (pathvarfound != 'WORKSPACE') { if (!pathvariableinfo.containsKey(pathvarfound)) { - println "Undefined path variable ${pathvarfound} found" + echo "Undefined path variable ${pathvarfound} found" exitcode = 1 } } } else { - println "Path: ${path} does not begin with a path variable" + echo "Path: ${path} does not begin with a path variable" exitcode = 1 } } if (exitcode > 0) { error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" + } else { + echo "ISM file: ${ISMfile} - ok" } } From 16ff70565612f9ef6590e8c4c3c928b6ed66e27e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 08:39:15 +0100 Subject: [PATCH 412/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 823c117..51edd3d 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -31,9 +31,8 @@ def placeBinaries(artifactDetails, installerDetails) { } @NonCPS -def checkISMfile(String ISMfile) { - def xml = readFile(ISMfile) - echo "Checking ISM file: ${ISMfile}" +def checkISMfilebackend(String xml) { + def String message = "" def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] @@ -54,26 +53,31 @@ def checkISMfile(String ISMfile) { } // Check that each path variable is defined beginning with a path variable or the WORKSPACE one pathvariableinfo.each{ pair -> - echo "pair key: ${pair.key} value: ${pair.value}" + message += "pair key: ${pair.key} value: ${pair.value}\n" def path = pair.value def pathvarmatcher = path =~ /<(.*)>.*/ if (pathvarmatcher.matches()) { def pathvarfound = pathvarmatcher.group(1) if (pathvarfound != 'WORKSPACE') { if (!pathvariableinfo.containsKey(pathvarfound)) { - echo "Undefined path variable ${pathvarfound} found" + message += "Undefined path variable ${pathvarfound} found\n" exitcode = 1 } } } else { - echo "Path: ${path} does not begin with a path variable" + message += "Path: ${path} does not begin with a path variable\n" exitcode = 1 } } + return [exitcode, message] +} + +def checkISMfile(String ISMfile) { + echo "Checking ISM file: ${ISMfile}" + def xml = readFile(ISMfile) + (exitcode, message) = checkISMfilebackend(xml) if (exitcode > 0) { - error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" - } else { - echo "ISM file: ${ISMfile} - ok" + error "ISM file errors found:\n${message}" } } From 291b39bf5bd2e05374a2fe136ca5476747df62e5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 08:57:32 +0100 Subject: [PATCH 413/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 51edd3d..6ac2e80 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -42,10 +42,10 @@ def checkISMfilebackend(String xml) { // Ignore the workspace definition which should have the only absolute path parsedxml.each{ table -> if (table.@name == "ISPathVariable") { - table.row.each{ row -> - def pathname = row.td[0].value()[0] - def path = row.td[1].value()[0] - if (path && pathname != "WORKSPACE") { + table['row'].each{ row -> + def pathname = row['td'][0].value()[0] + def path = row['td'][1].value()[0] + if (path && pathname != 'WORKSPACE') { pathvariableinfo[pathname] = path } } From 429d901d014ef6cd5669d9c0c078f680bd1c8649 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 09:30:24 +0100 Subject: [PATCH 414/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6ac2e80..ca9c502 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -78,6 +78,9 @@ def checkISMfile(String ISMfile) { (exitcode, message) = checkISMfilebackend(xml) if (exitcode > 0) { error "ISM file errors found:\n${message}" + } else { + echo "Checked ISM file: ${ISMfile} - ok" + echo message } } From 098b0cb8372189e4c30c6b55fe56b0ac4c7725c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 10:44:43 +0100 Subject: [PATCH 415/589] Perform installer file (ISM) checking --- install/installhelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index ca9c502..1464e9d 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -76,11 +76,11 @@ def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) (exitcode, message) = checkISMfilebackend(xml) + echo message if (exitcode > 0) { - error "ISM file errors found:\n${message}" + error "ISM file errors found" } else { - echo "Checked ISM file: ${ISMfile} - ok" - echo message + echo "Checked ISM file - ok" } } From 49a38e0eb1930649c7aa61a4cda9f64cb23a0f42 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 12 Apr 2019 14:55:49 +0100 Subject: [PATCH 416/589] Remove diff folder before building --- copy/logic.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..0b4823d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -14,6 +14,7 @@ pipeline { cleanfolders = [ "${WORKSPACE}/logic", "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/diff", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) From 3286473a46b80cb90aaeba9fd13c24e13a540821 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 15 Apr 2019 09:57:18 +0100 Subject: [PATCH 417/589] Logic copy doesn't work if pre-existing folders from previous builds exist --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..5a0c80d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,7 +36,7 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf meta + rm -v -rf diff meta embedded_logic mv -v logic/* . """ archiveArtifacts "${artifactlist}" From 5daf7c01d8239569439ea5ed3ab0f7e53460e0c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 21 Apr 2019 11:28:51 +0100 Subject: [PATCH 418/589] Meta-diff folder is not being reset at the beginning of a build --- meta/folder.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 346b2ee..6e0823e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -655,6 +655,10 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } + static String difffolder() { + return DIFFFOLDERNAME + } + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" @@ -763,6 +767,11 @@ MetaInfo metareset(script) { dir(mi.folder()) { deleteDir() } + + dir(mi.difffolder()) { + deleteDir() + } + return mi } From 46105a841d748708cb18a9627d7081744ce5964d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Apr 2019 22:47:29 +0100 Subject: [PATCH 419/589] GIS-7565 - Attempt at altering artifact output structure in order to simplify publishable item presentation to CAMs at publishing time --- release/do.groovy | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index fd2b06b..596f107 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -275,6 +275,22 @@ def sourcegitpush(String dryrunpush ) { } } +def boolean isalreadyreleased(buildmeta, tag) { + def boolean alreadyreleased = buildmeta.releaseinfo.released + if (alreadyreleased) { + echo "Component is already marked as released: ${alreadyreleased}" + } else { + String commitshacode = releasegitgetshacode(tag) + if (commitshacode) { + echo "Tag: ${tag} already exists in this repo therefore marking as already released" + alreadyreleased = true + } else { + alreadyreleased = false + } + } + return alreadyreleased +} + def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") def String buildstamptag = buildmeta.releasetag() @@ -283,8 +299,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - def boolean alreadyreleased = buildmeta.releaseinfo.released - echo "Already released: ${alreadyreleased}" + def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) def String postcommitshacode def boolean changed = false From 8cadf19ce95e36b8c6b4a89409d62d722a64ca06 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 26 Apr 2019 18:30:16 +0100 Subject: [PATCH 420/589] Slight change of message --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 596f107..9b46114 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -189,7 +189,7 @@ def boolean releasegittag(String postcommitshacode, String tag) { if (existingshacode) { // Tag exists if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + setunstable("The release tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") summaryappend("(${tag} - Could not add!)") } else { summaryappend("(${tag} - Already added)") From cbacd69e00c09bc24c21f473bbbbff9de27f1cc3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 09:38:53 +0100 Subject: [PATCH 421/589] Perform a git lfs install each time there is a git checkout --- versioncontrol/git.groovy | 58 +++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 253ffee..b386a9a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -75,7 +75,23 @@ def String checkouttoreporoot(String gitURL) { return reporootsubfolder } +def checkgitlfsinstall() { + if (isUnix()) { + sh """ + git lfs install + if [ \$? -ne 0 ]; then exit 1; fi + """ + + } else { + bat """ + git lfs install + if errorlevel 1 EXIT /b 1 + """ + } +} + def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', @@ -99,33 +115,33 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } String getSHAFromVersion(String versionFolder) { - dir (versionFolder) { + dir (versionFolder) { srcRevision = readFile("gitrevision.txt").trim() - } - return srcRevision + } + return srcRevision } boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def keys = versions.keySet() - if (keys.size() < 1) { - return false - } - if (keys.size() == 1) { - return true - } + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } else { - version = "" - for (key in keys) { - nextversion = versions.get(key) - if (version == "") { - version = nextversion - } - if (nextversion != version) { - return false - } - } - } - return true + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true } def stripgithistory(String rootfolder) { From e12939d9287e9a8f24c6ebb83d3a93a4007fb523 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 11:56:15 +0100 Subject: [PATCH 422/589] Added a comment --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 9b46114..b3d54c7 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -61,7 +61,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { - // Clear all existing files + // Clear all existing files - Note that this does not remove .git* sh("git rm -r *") } return previousreleasemeta From e00f228c7b9a5bdefa3eeefa1e929f473e4d2591 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 14:20:28 +0100 Subject: [PATCH 423/589] Detect if a publishable item filepath is valid or not --- publishing/utils.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 145a7e9..2ab4059 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,8 +1,23 @@ + +// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout +def checkallowed(String filepath) { + if (filepath.endsWith('/**') || filepath.endsWith('/')) { + def files = findFiles glob: filepath + if (files.length > 0) { + return + } else { + error "There are no files/folders under the publishable item: ${filepath}" + } + } + error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" +} + // Generate a list of artifact files that are publishable to outside parties def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { + checkallowed(filepath) meta.addPublishable(filepath) } } From 8fecc6c3200bb0643b33ac9b44b78f43b7545cc6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 14:50:31 +0100 Subject: [PATCH 424/589] Move functionality to common area --- install/installhelper.groovy | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 1464e9d..32da99b 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,3 +1,27 @@ +import hudson.FilePath + +// Transfer file from intermediate position to final position; +// creating necessary folders on the way; +// deleting intermediate file +def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { + def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() + + def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" + def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" + def export_filepath = new hudson.FilePath(channel, abs_export_out) + def export_out_folder = export_filepath.getParent() + def export_out_folder_string = export_out_folder.toString() + def export_out_file = export_filepath.getName() + + echo "Creating folder ${export_out_folder_string}" + export_out_folder.mkdirs() + + echo "Copying from ${pathToIntermediate} to ${abs_export_out}" + def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) + intermediate_filepath.copyTo(export_filepath) + intermediate_filepath.delete() +} + // Given details about artifacts to import and details of installers to build, import artifacts that are required def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] From c4651ca04e4c18cacb292be662a19681712b11b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 15:13:20 +0100 Subject: [PATCH 425/589] Release list is not being passed correctly --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6e0823e..b16db5d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -233,6 +233,7 @@ class ReleaseInfo extends GISJsonSerialised { } void addReleaseItem(String releaseitem) { + script.echo("Adding release item: ${releaseitem} to release items") releaseitems.add(releaseitem) } From 91a58544c3e8af3870b1200270c8cf90c5174b94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 28 Apr 2019 15:36:29 +0100 Subject: [PATCH 426/589] Remove redundant binaries copy to bin folder stage --- install/installhelper.groovy | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 32da99b..eebbd34 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -38,22 +38,6 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { return artifactFetched } -// Export any required binaries to the /bin folder -def placeBinaries(artifactDetails, installerDetails) { - for (installer in toBuild) { - (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) - if (exportBinaries) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) - for (binaryFolder in binariesFolders) { - def binariesSourceFolder = "${destFolder}/${binaryFolder}" - def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" - echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" - fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) - } - } - } -} - @NonCPS def checkISMfilebackend(String xml) { def String message = "" From 730ec0c5ff05fcec74bc1b7b1c432d07f9058df5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Apr 2019 10:55:10 +0100 Subject: [PATCH 427/589] Don't check publishable file paths for meta/** as its only save usually at the end of the build --- publishing/utils.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 2ab4059..2daf6f6 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,7 +1,7 @@ // The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout -def checkallowed(String filepath) { +def allowed(String filepath) { if (filepath.endsWith('/**') || filepath.endsWith('/')) { def files = findFiles glob: filepath if (files.length > 0) { @@ -17,7 +17,9 @@ def checkallowed(String filepath) { def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { - checkallowed(filepath) + if (filepath != meta.METAARTIFACTS) { + allowed(filepath) + } meta.addPublishable(filepath) } } From d55d2a24c4ea87e35a3ebae26f402807f7b10819 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:13:08 +0100 Subject: [PATCH 428/589] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- release/registerrepo.groovy | 2 +- versioncontrol/git.groovy | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index 2dffdc7..bd6a528 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -4,7 +4,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix) + vc.checkRepoExists(releaserepo) } meta.releaseinfo.set(releaserepo) } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index b386a9a..96fa571 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -114,6 +114,28 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo ]) } +def checkRepoExists(String gitURL) { + def boolean result = false + if (isUnix()) { + result = sh( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + } else { + result = bash( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + if errorlevel 1 EXIT /b 1 + """, + returnStatus: true + ) + } + if (!result) { + error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } +} + String getSHAFromVersion(String versionFolder) { dir (versionFolder) { srcRevision = readFile("gitrevision.txt").trim() From c13ba0d6289bd14c0a76198ca7125cfa859b3328 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:15:34 +0100 Subject: [PATCH 429/589] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 96fa571..cea31fc 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -124,7 +124,7 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } else { - result = bash( script: """ + result = bat( script: """ git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY if errorlevel 1 EXIT /b 1 """, From 9af654fb28e93396c128a571be4ab365fbd2bd45 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:20:36 +0100 Subject: [PATCH 430/589] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index cea31fc..36f4c64 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -119,14 +119,12 @@ def checkRepoExists(String gitURL) { if (isUnix()) { result = sh( script: """ git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true ) } else { result = bat( script: """ - git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - if errorlevel 1 EXIT /b 1 + @git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY """, returnStatus: true ) From a42acef027fd7fb3b8c4b47c9fb470584a27b75b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:26:22 +0100 Subject: [PATCH 431/589] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 36f4c64..12d5bc6 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -130,6 +130,7 @@ def checkRepoExists(String gitURL) { ) } if (!result) { + echo "git ls-remote return result: ${result}" error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") } } From 7a62550a07890fc69f415e36dcaf5c4f22627dd8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:29:15 +0100 Subject: [PATCH 432/589] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 12d5bc6..ee7e99a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -115,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } def checkRepoExists(String gitURL) { - def boolean result = false + def int result = 0 if (isUnix()) { result = sh( script: """ git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY @@ -129,7 +129,7 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } - if (!result) { + if (result > 0) { echo "git ls-remote return result: ${result}" error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") } From 0216685c99d3ce9c564a6c88fc61fd1f8bb99c22 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 3 May 2019 15:31:49 +0100 Subject: [PATCH 433/589] Rather than check out a release repo to see if it exists or not use git ls-remote to establish if it exists --- versioncontrol/git.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ee7e99a..f72fc5a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -129,9 +129,11 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } + echo "git ls-remote return result: ${result}" if (result > 0) { - echo "git ls-remote return result: ${result}" error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } else { + echo "The release repo: ${gitURL} exists" } } From 55a2e2a03444ed9e942316f3785667ba4f338375 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 7 May 2019 10:28:39 +0100 Subject: [PATCH 434/589] Warn of git stripping not being supported in Windows --- versioncontrol/git.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f72fc5a..b639c91 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -170,13 +170,17 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def stripgithistory(String rootfolder) { echo "Stripping git history out of ${rootfolder}" dir(rootfolder) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rf XX - find . -name ".gitignore" | xargs -iXX rm -rf XX - find . -name ".gitmodules" | xargs -iXX rm -rf XX - find . -iname ".gitattributes" | xargs -iXX rm -rf XX - """ + if (isUnix()) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + """ + } else { + echo "Not supporting git stripping in Windows at the moment" + } } } From b18dba3c00b34ce1202609d5947050950679d98b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 21:51:31 +0100 Subject: [PATCH 435/589] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index b639c91..fc15f81 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -93,6 +93,12 @@ def checkgitlfsinstall() { def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) + if (!sparseitemlist.isEmpty()) { + echo "Items in sparse item list: " + for (item in sparseitemlist) { + echo item + } + } checkout([ $class: 'GitSCM', userRemoteConfigs: [ From 4c4958db9b40be65fa611df7a5feb3e820873483 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 22:11:54 +0100 Subject: [PATCH 436/589] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index fc15f81..56d206c 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -94,10 +94,11 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) if (!sparseitemlist.isEmpty()) { - echo "Items in sparse item list: " + def itemstring = "Items in sparse item list:\n" for (item in sparseitemlist) { - echo item + itemstring += item + "\n" } + echo itemstring } checkout([ $class: 'GitSCM', @@ -114,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - sparsecheckoutconfig(sparseitemlist) + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], ], submoduleCfg: [], ]) From 5be5dfce45b25b7544fc9b0012f51905c66581d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 22:14:22 +0100 Subject: [PATCH 437/589] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 56d206c..5a92884 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -115,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'SparseCheckoutPaths', sparsecheckoutconfig(sparseitemlist)], ], submoduleCfg: [], ]) From ddc231ccacd21a672eaf1be65fa30640ba692178 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 22:54:47 +0100 Subject: [PATCH 438/589] Print out items in sparse item list if the list is not empty --- versioncontrol/git.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 5a92884..b37bbc7 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -11,12 +11,12 @@ String getGitInfo(String workingSetRootPath) { } @NonCPS -def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { +def ArrayList sparsecheckoutconfig(ArrayList paths) { ArrayList sparsecheckoutpaths = new ArrayList() for (path in paths) { sparsecheckoutpaths.add(new SparseCheckoutPath(path)) } - return new SparseCheckoutPaths(sparsecheckoutpaths) + return sparsecheckoutpaths } def String uniquereporootname(String gitURL) { @@ -115,7 +115,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparsecheckoutconfig(sparseitemlist)], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], ], submoduleCfg: [], ]) From 2c9113cc4d8045de29f6a9aeb33f9c5501904aab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 23:17:20 +0100 Subject: [PATCH 439/589] Print out items in sparse item list if the list is not empty --- publish/publish.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/publish/publish.groovy b/publish/publish.groovy index b478a5d..b74b72a 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -32,6 +32,15 @@ pipeline { } } } + stage('Test sparse checkout') { + steps { + script { + def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + def ArrayList publishablepaths = ['installers/KM/'] + gh.gitCheckout('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone + } + } + } stage('Get release artifact meta') { steps { script { From 7b50a27c26a0b492ddd17160babbaaa40756ef2d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 13 May 2019 23:23:32 +0100 Subject: [PATCH 440/589] Print out items in sparse item list if the list is not empty --- publish/publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index b74b72a..6f9fd44 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -36,7 +36,7 @@ pipeline { steps { script { def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def ArrayList publishablepaths = ['installers/KM/'] + def ArrayList publishablepaths = ['installers/KM/*'] gh.gitCheckout('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone } } From 6b9f624770a3e093ffdde027ed228d9d0880b9d0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 09:11:38 +0100 Subject: [PATCH 441/589] Need an alternative way to checkout because git plugin is not working when using LFS and sparse checkout simultaneously --- versioncontrol/git.groovy | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index b37bbc7..fed039a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -90,6 +90,91 @@ def checkgitlfsinstall() { } } +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + checkgitlfsinstall() + def reporootpath = checkouttoreporoot(gitURL) + if (!sparseitemlist.isEmpty()) { + def itemstring = "Items in sparse item list:\n" + for (item in sparseitemlist) { + itemstring += item + "\n" + } + echo itemstring + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + ], + submoduleCfg: [], + ]) +} + +//No credentials specified +//Cloning the remote Git repository +//Cloning repository git@hydra:print-server-releases/print_server_whole.git +//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 +//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git +//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// No valid HEAD. Skipping the resetting +// > git clean -fdx # timeout=10 +// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_7" +// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 + + +// > git rev-parse --is-inside-work-tree # timeout=10 +// Fetching changes from the remote Git repository +// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// Resetting working tree +// > git reset --hard # timeout=10 +// > git clean -fdx # timeout=10 +// Using shallow fetch with depth 1 +// Fetching upstream changes from git@hydra:releases/print-server-installer.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 +// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 +// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git read-tree -mu HEAD # timeout=10 +// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_3" +// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 + +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { checkgitlfsinstall() def reporootpath = checkouttoreporoot(gitURL) From a6cdcccb123d651b62c93fcc1792c4aaa955fe7b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:21:35 +0100 Subject: [PATCH 442/589] Try to get sparse checkout to work with lfs --- publish/do.groovy | 2 +- versioncontrol/git.groovy | 96 +++++++++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b63b15c..1599ca8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -83,7 +83,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite } if (publishablepaths.size() > 0) { def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone + gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone } else { summaryappend("None ") diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index fed039a..1652e6b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -90,37 +90,6 @@ def checkgitlfsinstall() { } } -def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - checkgitlfsinstall() - def reporootpath = checkouttoreporoot(gitURL) - if (!sparseitemlist.isEmpty()) { - def itemstring = "Items in sparse item list:\n" - for (item in sparseitemlist) { - itemstring += item + "\n" - } - echo itemstring - } - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: revision] - ], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], - ], - submoduleCfg: [], - ]) -} - //No credentials specified //Cloning the remote Git repository //Cloning repository git@hydra:print-server-releases/print_server_whole.git @@ -174,10 +143,12 @@ def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repo // Commit message: "2.10180.7389.21125_3" // > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 -// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { +def String lfsinstallandreporoot(String gitURL) { checkgitlfsinstall() - def reporootpath = checkouttoreporoot(gitURL) + return checkouttoreporoot(gitURL) +} + +def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { if (!sparseitemlist.isEmpty()) { def itemstring = "Items in sparse item list:\n" for (item in sparseitemlist) { @@ -185,6 +156,9 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo } echo itemstring } +} + +def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -206,6 +180,60 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo ]) } +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + +def gitinit(String targetDir, String reporootpath) { + if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + if (isUnix()) { + sh """ + git init "${targetDir}" + """ + } else { + bat """ + git init "${targetDir}" + """ + } + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") + } + } +} + +def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { + def String sparseitemliststring = "" + if (!sparseitemlist.isEmpty()) { + for (item in sparseitemlist) { + sparseitemliststring += "${item}," + } + dir(targetDir) { + if (isUnix()) { + sh """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } else { + bat """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } + } + } +} + +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitinit(targetDir, reporootpath) + setlfsfetchinclude(targetDir, sparseitemlist) + } + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + def checkRepoExists(String gitURL) { def int result = 0 if (isUnix()) { From 3686e219763bdaaf9608366e0bc57bfc5e63c4f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:32:18 +0100 Subject: [PATCH 443/589] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1652e6b..a5890f2 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -189,6 +189,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo def gitinit(String targetDir, String reporootpath) { if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + echo "git init ${targetDir}" if (isUnix()) { sh """ git init "${targetDir}" @@ -198,6 +199,7 @@ def gitinit(String targetDir, String reporootpath) { git init "${targetDir}" """ } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" dir("${targetDir}/.git/objects/info") { writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") } @@ -210,6 +212,7 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { for (item in sparseitemlist) { sparseitemliststring += "${item}," } + echo "git config lfs.fetchinclude ${sparseitemliststring}" dir(targetDir) { if (isUnix()) { sh """ @@ -225,6 +228,7 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { } def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) showsparseitemlist(sparseitemlist) if (!sparseitemlist.isEmpty()) { From f29e1beb7b0774d09dbc87d8d5810b3e8ae65bc5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:34:52 +0100 Subject: [PATCH 444/589] Try to get sparse checkout to work with lfs --- publish/publish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index 6f9fd44..e4539f1 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -37,7 +37,7 @@ pipeline { script { def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def ArrayList publishablepaths = ['installers/KM/*'] - gh.gitCheckout('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone + gh.gitCheckoutAlt('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone } } } From 1aee93e12163bc61a6dbacadf895674519b2fae0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 14 May 2019 23:42:49 +0100 Subject: [PATCH 445/589] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a5890f2..1bf9cd1 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -199,10 +199,10 @@ def gitinit(String targetDir, String reporootpath) { git init "${targetDir}" """ } - echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" - dir("${targetDir}/.git/objects/info") { - writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") - } + } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") } } From eb8a1005ec7f2e4f62e65a76f9386d72e3514699 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 May 2019 00:02:48 +0100 Subject: [PATCH 446/589] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1bf9cd1..ccbd458 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -227,6 +227,25 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { } } +def gitsparserestore(String targetDir) { + echo "Clean out full fetch; Force a sparse fetch" + dir(targetDir) { + if (isUnix()) { + sh """ + git rm -rf . + git clean -fxd + git reset --hard + """ + } else { + bat """ + git rm -rf . + git clean -fxd + git reset --hard + """ + } + } + } + def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) @@ -236,6 +255,9 @@ def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repo setlfsfetchinclude(targetDir, sparseitemlist) } docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitsparserestore(targetDir) + } } def checkRepoExists(String gitURL) { From 35ef539f5c85632660016fcbd60a58179001894b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 May 2019 00:16:04 +0100 Subject: [PATCH 447/589] Try to get sparse checkout to work with lfs --- versioncontrol/git.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ccbd458..8886076 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -232,15 +232,15 @@ def gitsparserestore(String targetDir) { dir(targetDir) { if (isUnix()) { sh """ - git rm -rf . - git clean -fxd - git reset --hard + git rm -rf . > /dev/null + git clean -fxd > /dev/null + git reset --hard > /dev/null """ } else { bat """ - git rm -rf . - git clean -fxd - git reset --hard + git rm -rf . 2> nul + git clean -fxd 2> nul + git reset --hard 2> nul """ } } From c1ceeca1430b9d90f792572c38291bbc7ea60aab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 15 May 2019 00:22:01 +0100 Subject: [PATCH 448/589] Try to get sparse checkout to work with lfs --- publish/publish.groovy | 9 --------- 1 file changed, 9 deletions(-) diff --git a/publish/publish.groovy b/publish/publish.groovy index e4539f1..b478a5d 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -32,15 +32,6 @@ pipeline { } } } - stage('Test sparse checkout') { - steps { - script { - def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def ArrayList publishablepaths = ['installers/KM/*'] - gh.gitCheckoutAlt('git@hydra:releases/print-server-installer.git', 'origin/master', 'testfolder', "", 1, true, publishablepaths) // Shallow clone - } - } - } stage('Get release artifact meta') { steps { script { From 26650ee254403cc2d6a850be1241b9000e124530 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:27:20 +0100 Subject: [PATCH 449/589] Add skeletal meta differences script files --- differences/dometahierarchy.groovy | 10 ++++ differences/metahierarchy.groovy | 92 ++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 differences/dometahierarchy.groovy create mode 100644 differences/metahierarchy.groovy diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy new file mode 100644 index 0000000..65e6cf4 --- /dev/null +++ b/differences/dometahierarchy.groovy @@ -0,0 +1,10 @@ + +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String summaryfolder, int maxdepth) { + def String summarystring = "No summary" + def boolean markunstable = true + def unstablereason = "No compare implementation yet" + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy new file mode 100644 index 0000000..3ea9dbc --- /dev/null +++ b/differences/metahierarchy.groovy @@ -0,0 +1,92 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + SUMMARYDIR = "${WORKSPACE}/summary" + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + "${WORKSPACE}/early", + "${WORKSPACE}/later", + "${SUMMARYDIR}", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout top level meta') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = 'git@hydra:gis/newsw.git' + + EARLYREVISION = "${RELEASE_EARLY_REVISION}".trim() + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) + + LATERREVISION = "${RELEASE_LATER_REVISION}".trim() + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) + } + } + } + stage('Diff components') { + steps { + script { + def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, SUMMARYDIR, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From f70406e72ba6ed5188134541454dd78a8f9561e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:35:45 +0100 Subject: [PATCH 450/589] Add skeletal meta differences script files --- differences/metahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 3ea9dbc..d82fcc0 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -29,14 +29,14 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - SOURCEURL = 'git@hydra:gis/newsw.git' + SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - EARLYREVISION = "${RELEASE_EARLY_REVISION}".trim() + EARLYREVISION = RELEASE_EARLY_REVISION.trim() EARLYROOTDIR = 'early' EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) - LATERREVISION = "${RELEASE_LATER_REVISION}".trim() + LATERREVISION = RELEASE_LATER_REVISION.trim() LATERROOTDIR = 'later' LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) From 36224b7c865a8218da05cc410ad96e2eddf50b86 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 16 May 2019 18:48:25 +0100 Subject: [PATCH 451/589] Add skeletal meta differences script files --- differences/dometahierarchy.groovy | 6 +++++- differences/metahierarchy.groovy | 2 +- release/do.groovy | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 65e6cf4..1d6885f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,5 +1,9 @@ -def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String summaryfolder, int maxdepth) { +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def int md = maxdepth.toInteger() + + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index d82fcc0..cd6738d 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -47,7 +47,7 @@ pipeline { steps { script { def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") - (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, SUMMARYDIR, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' diff --git a/release/do.groovy b/release/do.groovy index b3d54c7..738c04a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -365,7 +365,7 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri } def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { - int md = maxdepth.toInteger() + def int md = maxdepth.toInteger() resetfolders() preconditions() From 7e5c624d72bec8116849ac59e57cb75272981407 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 20 May 2019 11:02:28 +0100 Subject: [PATCH 452/589] Strip *@tmp folders from published folder --- versioncontrol/git.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8886076..534084d 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -323,6 +323,7 @@ def stripgithistory(String rootfolder) { find . -name ".gitignore" | xargs -iXX rm -rf XX find . -name ".gitmodules" | xargs -iXX rm -rf XX find . -iname ".gitattributes" | xargs -iXX rm -rf XX + find . -iname "*@tmp" | xargs -iXX rm -rf XX """ } else { echo "Not supporting git stripping in Windows at the moment" From 8961f250b17a89620fa0ea35b796d664512fdfb8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 19:39:52 +0100 Subject: [PATCH 453/589] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 23 ++++++++++++++++++++++- publish/do.groovy | 13 +++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 34707bc..9620e8a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,7 +42,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -76,6 +76,27 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(destpath) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..6207380 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,6 +148,19 @@ def convertitemstopublish(String itemstopublishstring) { return paths } +def convertstringitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { From e418c593be9bb8ba865142321491b846567c3a7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:04:39 +0100 Subject: [PATCH 454/589] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 9620e8a..d85818f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) From a5a2603e27dfd1e94c84b134a06d07d0dfc5f8f6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:25:53 +0100 Subject: [PATCH 455/589] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- publish/do.groovy | 13 ------------- versioncontrol/git.groovy | 13 +++++++++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index d85818f..764618c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) diff --git a/publish/do.groovy b/publish/do.groovy index 6207380..1599ca8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,19 +148,6 @@ def convertitemstopublish(String itemstopublishstring) { return paths } -def convertstringitemstopublish(String itemstopublishstring) { - def List paths = [] - - def String[] items = itemstopublishstring.split(',') - def String allpaths = "" - for(String item : items) { - paths.add(item) - allpaths += "${path}\n" - } - echo "Selected item to publish:\n${allpaths}" - return paths -} - // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 534084d..818734a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -246,6 +246,19 @@ def gitsparserestore(String targetDir) { } } +def List convertSparseItemsString(String itemstopublish) { + def List paths = [] + + def String[] items = itemstopublish.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) From f1cb5075a2794e74a09d58c5f50a118f9813eeff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:33:55 +0100 Subject: [PATCH 456/589] MCS-425 - Perform a sparse checkout of items to import from a release repo --- versioncontrol/git.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 818734a..811d072 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -247,16 +247,16 @@ def gitsparserestore(String targetDir) { } def List convertSparseItemsString(String itemstopublish) { - def List paths = [] + def List sparsepaths = [] def String[] items = itemstopublish.split(',') def String allpaths = "" for(String item : items) { - paths.add(item) - allpaths += "${path}\n" + sparsepaths.add(item) + allpaths += "${item}\n" } echo "Selected item to publish:\n${allpaths}" - return paths + return sparsepaths } def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { From 2c5caba8ec8d123c2e4f852a7c33baafce6723e2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 13:30:15 +0100 Subject: [PATCH 457/589] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- meta/folder.groovy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 764618c..fd43379 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -68,7 +68,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) diff --git a/meta/folder.groovy b/meta/folder.groovy index b16db5d..f8c033b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -491,6 +491,7 @@ class MetaInfo extends GISJsonSerialised { } void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") releaseinfo.markreleased() for (associated in associatedmeta) { associated.meta.releaseinfo.markreleased() From 633a8dc8cb9bd107300c238a539cf97a56b5bcec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 14:08:13 +0100 Subject: [PATCH 458/589] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index fd43379..09eb337 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -89,7 +89,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) From b1d34d8f8795d1c6629537a7a5a5bc9b911becb4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 28 Jun 2019 18:25:28 +0100 Subject: [PATCH 459/589] Make the default filter blank; Alter the order of build import --- import/importer.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 09eb337..418fd9a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -17,7 +17,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, bo return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -42,7 +42,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -76,7 +76,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d return importedmeta } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -97,7 +97,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) From 2907eb9dc5ed4ff20295a5ca4af4d816b1506553 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 09:54:24 +0100 Subject: [PATCH 460/589] Make the default filter blank; Alter the order of build import --- versioncontrol/git.groovy | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 811d072..7e065d5 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -249,13 +249,16 @@ def gitsparserestore(String targetDir) { def List convertSparseItemsString(String itemstopublish) { def List sparsepaths = [] - def String[] items = itemstopublish.split(',') - def String allpaths = "" - for(String item : items) { - sparsepaths.add(item) - allpaths += "${item}\n" + boolean nonempty = itemstopublish?.trim() + if (nonempty) { + def String[] items = itemstopublish..trim().split(',') + def String allpaths = "" + for(String item : items) { + sparsepaths.add(item) + allpaths += "${item}\n" + } + echo "Selected item to publish:\n${allpaths}" } - echo "Selected item to publish:\n${allpaths}" return sparsepaths } From 7577060104098b6daf10a463b7758ce9d29930fd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 10:49:46 +0100 Subject: [PATCH 461/589] Remove redundant references to temp folders --- publish/do.groovy | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..18ec404 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -8,7 +8,6 @@ summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" -tempfolder = "${WORKSPACE}/temp" def resetfolders() { dir(summaryfolder) { @@ -20,9 +19,6 @@ def resetfolders() { dir(filteredreleaserepofolder) { deleteDir() } - dir(tempfolder) { - deleteDir() - } } def productuniquefolder(releasemeta) { @@ -61,8 +57,7 @@ def guessreporootsuffix(componentURL) { } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { - // Copy relevant artifacts to temp folder +def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { summaryappend("Artifacts: ") def releasemeta = releaseinfo.meta @@ -124,11 +119,10 @@ def copydirectory(File dirFrom, File dirTo) { def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) - def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) summaryappend("\n") } From 575fade931e0c0e1e83adc7168c44cef606015d2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 12:59:46 +0100 Subject: [PATCH 462/589] Add an option to import from repo to import lower levels of artifacts --- import/importer.groovy | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 418fd9a..68b8e4c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,11 +42,14 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } +publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - gh.gitCheckout(rooturl, rootrevision, destpath) + + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) @@ -59,8 +62,11 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckout(cloneurl, clonerevision, clonepath) + + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" echo "Marking related meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(relatedmeta, metafilepath) @@ -90,18 +96,21 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean recursive = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) case "Release repository": + if (nested) { + return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) default: error("Unknown type of artifact source: ${source}") From d9cd1acbb8fc5196ecb51dfe2494653ec297fa87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 1 Jul 2019 13:19:05 +0100 Subject: [PATCH 463/589] Add an option to import from repo to import lower levels of artifacts --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 68b8e4c..ba5eb0b 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -103,7 +103,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean recursive = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) From f209f2a9d824ac0a4aac5da2f76af397d8237951 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Jul 2019 08:51:06 +0100 Subject: [PATCH 464/589] Further meta hierarchy difference work --- configuration/configuration.groovy | 2 +- differences/dometahierarchy.groovy | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 30b7086..1864ba9 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -9,7 +9,7 @@ def registerPipelineScriptConfiguration(meta) { } def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { - gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1d6885f..f5de8e8 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,3 +1,12 @@ +metafolderscript = load("${commonscripts()}/meta/folder.groovy") + +def showmetaitems(metaitems) + for (entry in metaitems) { + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath + echo "Name: ${name}\t\t\t\tPath: ${offsetpath}" + } +} def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def int md = maxdepth.toInteger() @@ -7,6 +16,18 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" + + def earlymeta = metafolderscript.importrelated(earlymetafolder) + def latermeta = metafolderscript.importrelated(latermetafolder) + + def earlymetaitems = earlymeta.releasewalkerunique(md) + def latermetaitems = latermeta.releasewalkerunique(md) + + echo "Early items:" + showmetaitems(earlymetaitems) + + echo "Later items:" + showmetaitems(latermetaitems) return [summarystring, markunstable, unstablereason] } From d68196c2c62eb7ae0c7c52783c6a726a06625cea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Jul 2019 13:26:33 +0100 Subject: [PATCH 465/589] Further meta hierarchy difference work --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f5de8e8..fa11a2d 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,6 +1,6 @@ metafolderscript = load("${commonscripts()}/meta/folder.groovy") -def showmetaitems(metaitems) +def showmetaitems(metaitems) { for (entry in metaitems) { def name = entry.meta.componentversionuniqueidentifier() def offsetpath = entry.offsetpath From 6cfd67289696fbc49560a91c2bc9f860f2402bca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 2 Jul 2019 13:32:54 +0100 Subject: [PATCH 466/589] Further meta hierarchy difference work --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index fa11a2d..7f60aae 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,4 +1,4 @@ -metafolderscript = load("${commonscripts()}/meta/folder.groovy") +metafolderscript = load("${autobuilder.commonscripts()}/meta/folder.groovy") def showmetaitems(metaitems) { for (entry in metaitems) { From ebcc588604e682ec6296fba51b312ab670ecfb3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 08:05:43 +0100 Subject: [PATCH 467/589] No need to import meta script - attempting to remove jenkins crash --- differences/dometahierarchy.groovy | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 7f60aae..b6c9660 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,5 +1,3 @@ -metafolderscript = load("${autobuilder.commonscripts()}/meta/folder.groovy") - def showmetaitems(metaitems) { for (entry in metaitems) { def name = entry.meta.componentversionuniqueidentifier() @@ -11,18 +9,18 @@ def showmetaitems(metaitems) { def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def int md = maxdepth.toInteger() - echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" def String summarystring = "No summary" def boolean markunstable = true def unstablereason = "No compare implementation yet" - - def earlymeta = metafolderscript.importrelated(earlymetafolder) - def latermeta = metafolderscript.importrelated(latermetafolder) - + + def earlymeta = meta.importrelated(earlymetafolder) + def latermeta = meta.importrelated(latermetafolder) + def earlymetaitems = earlymeta.releasewalkerunique(md) def latermetaitems = latermeta.releasewalkerunique(md) - + echo "Early items:" showmetaitems(earlymetaitems) From 8b4fbe2005f4a53731e8646d4a11cee00263a3ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 08:26:53 +0100 Subject: [PATCH 468/589] No need to import meta script - attempting to remove jenkins crash --- differences/dometahierarchy.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index b6c9660..62cd5ee 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,8 +1,8 @@ def showmetaitems(metaitems) { - for (entry in metaitems) { - def name = entry.meta.componentversionuniqueidentifier() - def offsetpath = entry.offsetpath - echo "Name: ${name}\t\t\t\tPath: ${offsetpath}" + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + def offsetpath = relatedmetainfo.offsetpath + echo "Name: ${key}\t\t\t\tPath: ${offsetpath}" } } From 01ff6c1a5f46f883662e1268b925c03f9d8a6b94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 08:47:33 +0100 Subject: [PATCH 469/589] No need to import meta script - attempting to remove jenkins crash --- differences/dometahierarchy.groovy | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 62cd5ee..1a6ed7f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,9 +1,23 @@ def showmetaitems(metaitems) { + String info = '' for (key in metaitems.keySet()) { def relatedmetainfo = metaitems[key] def offsetpath = relatedmetainfo.offsetpath - echo "Name: ${key}\t\t\t\tPath: ${offsetpath}" + info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" } + echo info +} + +def compare(earlymetaitems, latermetaitems, key) { + echo "Collecting changes between early and later versions of component: ${key}" +} + +def deletedcomponent(earlymetaitems, key) { + echo "Component ${key} has been deleted in later version" +} + +def addedcomponent(latermetaitems, key) { + echo "Component ${key} has been added in later version" } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { @@ -26,6 +40,19 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Later items:" showmetaitems(latermetaitems) + + for (key in earlymetaitems.keySet()) { + if (latermetaitems.containsKey(key)) { + compare(earlymetaitems, latermetaitems, key) + } else { + deletedcomponent(earlymetaitems, key) + } + } + for (key in latermetaitems.keySet()) { + if (!earlymetaitems.containsKey(key)) { + addedcomponent(latermetaitems, key) + } + } return [summarystring, markunstable, unstablereason] } From 22a5b7bde7460474ac9e1c9f148b837bac82bd0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 15:17:21 +0100 Subject: [PATCH 470/589] More infrastructure for differencing versions of component --- differences/do.groovy | 20 ++++++++ differences/dometahierarchy.groovy | 76 ++++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 32f18b5..1499e6d 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,4 +15,24 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def uniquefolderforsourcerepo = getcomponent(earlymeta) + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + } else { + echo "The early and later git urls don't match" + result = false; + } + + return [result, differencerecords] +} + return this; diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1a6ed7f..6ca7e08 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,3 +1,44 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" + +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + def showmetaitems(metaitems) { String info = '' for (key in metaitems.keySet()) { @@ -8,8 +49,33 @@ def showmetaitems(metaitems) { echo info } -def compare(earlymetaitems, latermetaitems, key) { +def String getcomponent(meta) { + def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + summaryappend("${meta.productinfo.productname} ${meta.productinfo.subcomponentname}: ") + + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + releaser.checkoutsourcerepo(meta, uniquefolderforsourcerepo) + + summaryappend("\n") + return uniquefolderforsourcerepo +} + +def boolean compare(earlymetaitems, latermetaitems, key) { + boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].meta + def latermeta = latermetaitems[key].meta + + def uniquefolderforsourcerepo = getcomponent(earlymeta) + + def differencer = load("${autobuilder.commonscripts()}/differences/do.groovy") + (result, differencerecords) = differencer.getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + return result; } def deletedcomponent(earlymetaitems, key) { @@ -22,12 +88,14 @@ def addedcomponent(latermetaitems, key) { def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def int md = maxdepth.toInteger() + resetfolders() + preconditions() echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - def String summarystring = "No summary" - def boolean markunstable = true - def unstablereason = "No compare implementation yet" + summarystring = "No summary" + markunstable = true + unstablereason = "No compare implementation yet" def earlymeta = meta.importrelated(earlymetafolder) def latermeta = meta.importrelated(latermetafolder) From 4dfde882fea490a115158a8ecb2b72508bb1b66a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 15:28:24 +0100 Subject: [PATCH 471/589] More infrastructure for differencing versions of component --- differences/dometahierarchy.groovy | 3 --- 1 file changed, 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6ca7e08..5047ce2 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -12,9 +12,6 @@ def resetfolders() { dir(summaryfolder) { deleteDir() } - dir(releaserepofolder) { - deleteDir() - } dir(tempfolder) { deleteDir() } From cc262c09ad51cedc1908203b99b760e65c9f5767 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 15:45:04 +0100 Subject: [PATCH 472/589] More infrastructure for differencing versions of component --- differences/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/differences/do.groovy b/differences/do.groovy index 1499e6d..6c5c947 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -23,7 +23,6 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def latergitinfo = latermeta.gitinfo if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { - def uniquefolderforsourcerepo = getcomponent(earlymeta) def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" From 437e27b0e2e67b31e1c03a28ba2c30a0741d8f7b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 16:09:14 +0100 Subject: [PATCH 473/589] More infrastructure for differencing versions of component --- differences/do.groovy | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/differences/do.groovy b/differences/do.groovy index 6c5c947..0bf3f4f 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,6 +15,20 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def differencerecords = [] + def String releaseexpression + if tripledot: + releaseexpression = earlyrevision + "..." + laterrevision + else: + releaseexpression = earlyrevision + ".." + laterrevision + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() + } + return differencetext +} + def getdifferencerecords(earlymeta, latermeta, String sourcepath) { boolean result = true; def List differencerecords = [] @@ -26,6 +40,8 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + echo "Difference text is:\n${differencetext}" } else { echo "The early and later git urls don't match" result = false; From 9458a9485a8be66e046e562974db7c3865d453ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 16:12:44 +0100 Subject: [PATCH 474/589] More infrastructure for differencing versions of component --- differences/do.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 0bf3f4f..eb4a53a 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -18,10 +18,11 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { def differencerecords = [] def String releaseexpression - if tripledot: + if (tripledot) { releaseexpression = earlyrevision + "..." + laterrevision - else: + } else { releaseexpression = earlyrevision + ".." + laterrevision + } def String differencetext dir(gitrootfolder) { differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() From 79405e223b00fb298bbb011abef8687e11058277 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:04:24 +0100 Subject: [PATCH 475/589] More infrastructure for differencing versions of component --- differences/do.groovy | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/differences/do.groovy b/differences/do.groovy index eb4a53a..99596f3 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,6 +15,20 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } +class DifferenceRecord { + String productname + String subcomponentname + String jirakey + + String rawdifferencetext + String sha + int type + String jiraticketinfo + + DifferenceRecord() { + } +} + def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { def differencerecords = [] def String releaseexpression @@ -30,6 +44,26 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late return differencetext } +def List processtext(meta, String text) { + def List differencerecords = [] + + text.eachLine { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = it.trim() + dr.sha = "sha" + dr.type = 0 + dr.jiraticketinfo = "info" + + differencerecords.add(dr) + } + return differencerecords +} + def getdifferencerecords(earlymeta, latermeta, String sourcepath) { boolean result = true; def List differencerecords = [] @@ -43,6 +77,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) echo "Difference text is:\n${differencetext}" + differencerecords = differencerecords + processtext(earlymeta, differencetext) } else { echo "The early and later git urls don't match" result = false; From d202e0bb3e4460735385f25a1f32bdb45cac9970 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:19:12 +0100 Subject: [PATCH 476/589] More infrastructure for differencing versions of component --- differences/do.groovy | 11 +++++++++-- differences/dometahierarchy.groovy | 13 +++++++++---- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 99596f3..b29aaa4 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -16,6 +16,8 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } class DifferenceRecord { + def script + String productname String subcomponentname String jirakey @@ -25,7 +27,12 @@ class DifferenceRecord { int type String jiraticketinfo - DifferenceRecord() { + DifferenceRecord(script) { + this.script = script + } + + def dump() { + script.echo("Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" } } @@ -48,7 +55,7 @@ def List processtext(meta, String text) { def List differencerecords = [] text.eachLine { - def DifferenceRecord dr = new DifferenceRecord() + def DifferenceRecord dr = new DifferenceRecord(this) dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5047ce2..464dce9 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -60,7 +60,7 @@ def String getcomponent(meta) { return uniquefolderforsourcerepo } -def boolean compare(earlymetaitems, latermetaitems, key) { +def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" @@ -72,7 +72,7 @@ def boolean compare(earlymetaitems, latermetaitems, key) { def differencer = load("${autobuilder.commonscripts()}/differences/do.groovy") (result, differencerecords) = differencer.getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - return result; + return [result, differencerecords]; } def deletedcomponent(earlymetaitems, key) { @@ -84,6 +84,8 @@ def addedcomponent(latermetaitems, key) { } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def List differencerecords = [] + def int md = maxdepth.toInteger() resetfolders() preconditions() @@ -108,7 +110,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { - compare(earlymetaitems, latermetaitems, key) + (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + differencerecords = differencerecords + componentdifferencerecords } else { deletedcomponent(earlymetaitems, key) } @@ -118,7 +121,9 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin addedcomponent(latermetaitems, key) } } - + for (record in differencerecords) { + record.dump() + } return [summarystring, markunstable, unstablereason] } From 28d24268fb9a78a98ac492092fe983d1a40f99ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:29:03 +0100 Subject: [PATCH 477/589] Add more difference infrastructure --- differences/do.groovy | 6 ++---- differences/dometahierarchy.groovy | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index b29aaa4..0de5395 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -16,8 +16,6 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } class DifferenceRecord { - def script - String productname String subcomponentname String jirakey @@ -31,8 +29,8 @@ class DifferenceRecord { this.script = script } - def dump() { - script.echo("Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" + def String dump() { + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" } } diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 464dce9..2770c4a 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -122,7 +122,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin } } for (record in differencerecords) { - record.dump() + echo "${record.dump()}" } return [summarystring, markunstable, unstablereason] } From f00082dd8b3fa3daf6e65669cb48d4fba4e99076 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 18:51:01 +0100 Subject: [PATCH 478/589] Add more difference infrastructure --- differences/do.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 0de5395..611ce68 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -25,8 +25,7 @@ class DifferenceRecord { int type String jiraticketinfo - DifferenceRecord(script) { - this.script = script + DifferenceRecord() { } def String dump() { @@ -53,7 +52,7 @@ def List processtext(meta, String text) { def List differencerecords = [] text.eachLine { - def DifferenceRecord dr = new DifferenceRecord(this) + def DifferenceRecord dr = new DifferenceRecord() dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname From 02b104312dceca59b681c8911501b3593c070888 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 19:34:26 +0100 Subject: [PATCH 479/589] Add more difference infrastructure --- differences/do.groovy | 75 -------------------- differences/dometahierarchy.groovy | 106 +++++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 79 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 611ce68..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -15,79 +15,4 @@ def differences(meta, String sourcerevision, String previoussourcerevision, Stri } } -class DifferenceRecord { - String productname - String subcomponentname - String jirakey - - String rawdifferencetext - String sha - int type - String jiraticketinfo - - DifferenceRecord() { - } - - def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" - } -} - -def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { - def differencerecords = [] - def String releaseexpression - if (tripledot) { - releaseexpression = earlyrevision + "..." + laterrevision - } else { - releaseexpression = earlyrevision + ".." + laterrevision - } - def String differencetext - dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() - } - return differencetext -} - -def List processtext(meta, String text) { - def List differencerecords = [] - - text.eachLine { - def DifferenceRecord dr = new DifferenceRecord() - - dr.productname = meta.productinfo.productname - dr.subcomponentname = meta.productinfo.subcomponentname - dr.jirakey = meta.productinfo.jirakey - - dr.rawdifferencetext = it.trim() - dr.sha = "sha" - dr.type = 0 - dr.jiraticketinfo = "info" - - differencerecords.add(dr) - } - return differencerecords -} - -def getdifferencerecords(earlymeta, latermeta, String sourcepath) { - boolean result = true; - def List differencerecords = [] - - def earlygitinfo = earlymeta.gitinfo - def latergitinfo = latermeta.gitinfo - - if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { - def earlyrevision = earlygitinfo.gitrevision - def laterrevision = latergitinfo.gitrevision - echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" - def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - echo "Difference text is:\n${differencetext}" - differencerecords = differencerecords + processtext(earlymeta, differencetext) - } else { - echo "The early and later git urls don't match" - result = false; - } - - return [result, differencerecords] -} - return this; diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 2770c4a..f60efd7 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -8,6 +8,24 @@ summarystring = "" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" +class DifferenceRecord { + String productname + String subcomponentname + String jirakey + + String rawdifferencetext + String sha + int type + String jiraticketinfo + + DifferenceRecord() { + } + + def String dump() { + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" + } +} + def resetfolders() { dir(summaryfolder) { deleteDir() @@ -46,6 +64,31 @@ def showmetaitems(metaitems) { echo info } +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } +} + def String getcomponent(meta) { def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -53,13 +96,69 @@ def String getcomponent(meta) { summaryappend("${meta.productinfo.productname} ${meta.productinfo.subcomponentname}: ") - def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") - releaser.checkoutsourcerepo(meta, uniquefolderforsourcerepo) + checkoutsourcerepo(meta, uniquefolderforsourcerepo) summaryappend("\n") return uniquefolderforsourcerepo } +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def differencerecords = [] + def String releaseexpression + if (tripledot) { + releaseexpression = earlyrevision + "..." + laterrevision + } else { + releaseexpression = earlyrevision + ".." + laterrevision + } + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() + } + return differencetext +} + +def List processtext(meta, String text) { + def List differencerecords = [] + + text.eachLine { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = it.trim() + dr.sha = "sha" + dr.type = 0 + dr.jiraticketinfo = "info" + + differencerecords.add(dr) + } + return differencerecords +} + +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + echo "Difference text is:\n${differencetext}" + differencerecords = processtext(earlymeta, differencetext) + } else { + echo "The early and later git urls don't match" + result = false; + } + + return [result, differencerecords] +} + def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" @@ -69,8 +168,7 @@ def compare(earlymetaitems, latermetaitems, key) { def uniquefolderforsourcerepo = getcomponent(earlymeta) - def differencer = load("${autobuilder.commonscripts()}/differences/do.groovy") - (result, differencerecords) = differencer.getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) return [result, differencerecords]; } From 665e0039adb0f3a5a64efd0f77a3f9e763f9d523 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 19:52:53 +0100 Subject: [PATCH 480/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f60efd7..9e166b7 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -120,9 +120,11 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def List processtext(meta, String text) { def List differencerecords = [] + record = 0; text.eachLine { def DifferenceRecord dr = new DifferenceRecord() + echo "Creating record: ${record}" dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname dr.jirakey = meta.productinfo.jirakey @@ -133,6 +135,7 @@ def List processtext(meta, String text) { dr.jiraticketinfo = "info" differencerecords.add(dr) + record += 1 } return differencerecords } @@ -169,7 +172,7 @@ def compare(earlymetaitems, latermetaitems, key) { def uniquefolderforsourcerepo = getcomponent(earlymeta) (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - + return [result, differencerecords]; } @@ -209,6 +212,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + echo "Returned: ${result}, records: ${componentdifferencerecords.size()}" + differencerecords = differencerecords + componentdifferencerecords } else { deletedcomponent(earlymetaitems, key) @@ -219,6 +224,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin addedcomponent(latermetaitems, key) } } + echo "Result: ${result}, total records: ${differencerecords.size()}" for (record in differencerecords) { echo "${record.dump()}" } From c75e58cc697dac6bed9d4bcaf4ba7277a0b30d72 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 3 Jul 2019 19:59:31 +0100 Subject: [PATCH 481/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9e166b7..97efa08 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -121,7 +121,7 @@ def List processtext(meta, String text) { def List differencerecords = [] record = 0; - text.eachLine { + text.split('\n').each { def DifferenceRecord dr = new DifferenceRecord() echo "Creating record: ${record}" From fe72ba4a9f1b4e210508f2afff1cd3e18c86d60f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 08:40:22 +0100 Subject: [PATCH 482/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 97efa08..dc35bdf 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -94,11 +94,8 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - summaryappend("${meta.productinfo.productname} ${meta.productinfo.subcomponentname}: ") - checkoutsourcerepo(meta, uniquefolderforsourcerepo) - summaryappend("\n") return uniquefolderforsourcerepo } @@ -120,11 +117,9 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def List processtext(meta, String text) { def List differencerecords = [] - record = 0; text.split('\n').each { def DifferenceRecord dr = new DifferenceRecord() - echo "Creating record: ${record}" dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname dr.jirakey = meta.productinfo.jirakey @@ -135,7 +130,6 @@ def List processtext(meta, String text) { dr.jiraticketinfo = "info" differencerecords.add(dr) - record += 1 } return differencerecords } @@ -151,12 +145,14 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - echo "Difference text is:\n${differencetext}" differencerecords = processtext(earlymeta, differencetext) } else { + summaryappend("The early and later git urls don't match") echo "The early and later git urls don't match" result = false; + setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") } return [result, differencerecords] @@ -165,6 +161,7 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" + summaryappend("Changed: ") def earlymeta = earlymetaitems[key].meta def latermeta = latermetaitems[key].meta @@ -178,10 +175,12 @@ def compare(earlymetaitems, latermetaitems, key) { def deletedcomponent(earlymetaitems, key) { echo "Component ${key} has been deleted in later version" + summaryappend("Deleted ") } def addedcomponent(latermetaitems, key) { echo "Component ${key} has been added in later version" + summaryappend("Added ") } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { @@ -193,10 +192,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - summarystring = "No summary" - markunstable = true - unstablereason = "No compare implementation yet" - def earlymeta = meta.importrelated(earlymetafolder) def latermeta = meta.importrelated(latermetafolder) @@ -210,21 +205,26 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin showmetaitems(latermetaitems) for (key in earlymetaitems.keySet()) { + def earlymeta = earlymetaitems[key].meta if (latermetaitems.containsKey(key)) { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) - echo "Returned: ${result}, records: ${componentdifferencerecords.size()}" - differencerecords = differencerecords + componentdifferencerecords + summaryappend("\n") } else { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) + summaryappend("\n") } } for (key in latermetaitems.keySet()) { + def latermeta = latermetaitems[key].meta if (!earlymetaitems.containsKey(key)) { + summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) + summaryappend("\n") } } - echo "Result: ${result}, total records: ${differencerecords.size()}" for (record in differencerecords) { echo "${record.dump()}" } From 319ae116ca3af419130919ce71b136f43b1d5af4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 09:06:52 +0100 Subject: [PATCH 483/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dc35bdf..0b42152 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -205,7 +205,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin showmetaitems(latermetaitems) for (key in earlymetaitems.keySet()) { - def earlymeta = earlymetaitems[key].meta if (latermetaitems.containsKey(key)) { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) @@ -218,7 +217,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin } } for (key in latermetaitems.keySet()) { - def latermeta = latermetaitems[key].meta if (!earlymetaitems.containsKey(key)) { summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) From d1d69c6c63c9f2aa9cd7bd8d996264dab1828804 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:03:16 +0100 Subject: [PATCH 484/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 45 ++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 0b42152..bbfde2b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -114,21 +114,42 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late return differencetext } +def extractsha(String line) { + def pattern = ~"^(.*)\s(.*)" + + def sha = line.replaceFirst(pattern) { _, sha, therest -> "${sha.trim()}" } + def therest = line.replaceFirst(pattern) { _, sha, therest -> "${therest.trim()}" } + + echo "Sha filtering: line: ${line} sha: ${sha} therest: ${therest}" + + return [sha, therest] +} + +def DifferenceRecord fillrecord(meta, line) { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = line + + // Example of raw text to process: + // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py + + (dr.sha, line) = extractsha(line) + + dr.type = 0 + dr.jiraticketinfo = "info" + + return dr +} + def List processtext(meta, String text) { def List differencerecords = [] - - text.split('\n').each { - def DifferenceRecord dr = new DifferenceRecord() - - dr.productname = meta.productinfo.productname - dr.subcomponentname = meta.productinfo.subcomponentname - dr.jirakey = meta.productinfo.jirakey - dr.rawdifferencetext = it.trim() - dr.sha = "sha" - dr.type = 0 - dr.jiraticketinfo = "info" - + text.split('\n').each { + DifferenceRecord dr = fillrecord(meta, it.trim()) differencerecords.add(dr) } return differencerecords From a08335c68eb0712f9266182b84d15e6a9a05754b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:07:47 +0100 Subject: [PATCH 485/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index bbfde2b..395b478 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -115,7 +115,7 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late } def extractsha(String line) { - def pattern = ~"^(.*)\s(.*)" + def pattern = ~"^(.*)\\s(.*)" def sha = line.replaceFirst(pattern) { _, sha, therest -> "${sha.trim()}" } def therest = line.replaceFirst(pattern) { _, sha, therest -> "${therest.trim()}" } From a9ff43ff0aa8922e76f622218cdb18ddd579f35d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:14:53 +0100 Subject: [PATCH 486/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 395b478..912583d 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -137,7 +137,8 @@ def DifferenceRecord fillrecord(meta, line) { // Example of raw text to process: // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py - (dr.sha, line) = extractsha(line) + (sha, line) = extractsha(line) + dr.sha = sha dr.type = 0 dr.jiraticketinfo = "info" From a457787eeab03a0baa1cef505244f61357cec648 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 11:18:32 +0100 Subject: [PATCH 487/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 912583d..5350529 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -117,10 +117,10 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def extractsha(String line) { def pattern = ~"^(.*)\\s(.*)" - def sha = line.replaceFirst(pattern) { _, sha, therest -> "${sha.trim()}" } - def therest = line.replaceFirst(pattern) { _, sha, therest -> "${therest.trim()}" } + def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit.trim()}" } + def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest.trim()}" } - echo "Sha filtering: line: ${line} sha: ${sha} therest: ${therest}" + echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" return [sha, therest] } From ca296aac27be9194802b150a9ec9a3d6e15c9298 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 13:05:37 +0100 Subject: [PATCH 488/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5350529..f98b96f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -115,14 +115,14 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late } def extractsha(String line) { - def pattern = ~"^(.*)\\s(.*)" + def pattern = ~"^(\\p{XDigit}*) (.*)" - def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit.trim()}" } - def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest.trim()}" } + def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit}" } + def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest}" } echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" - return [sha, therest] + return [sha.trim(), therest.trim()] } def DifferenceRecord fillrecord(meta, line) { @@ -137,7 +137,7 @@ def DifferenceRecord fillrecord(meta, line) { // Example of raw text to process: // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py - (sha, line) = extractsha(line) + (sha, line2) = extractsha(line) dr.sha = sha dr.type = 0 From 90ed5663c64016860a6aba5883086a20bf6eef68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 13:08:43 +0100 Subject: [PATCH 489/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f98b96f..a58c02e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -122,7 +122,7 @@ def extractsha(String line) { echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" - return [sha.trim(), therest.trim()] + return [sha.trim(), restofline.trim()] } def DifferenceRecord fillrecord(meta, line) { From d51458d7b45d64a03305f1d30684f788d5a1e3e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 13:26:46 +0100 Subject: [PATCH 490/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a58c02e..583c7f1 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -116,9 +116,11 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late def extractsha(String line) { def pattern = ~"^(\\p{XDigit}*) (.*)" + def match = (line =~ pattern) + match.find() - def sha = line.replaceFirst(pattern) { _, shabit, therest -> "${shabit}" } - def restofline = line.replaceFirst(pattern) { _, shabit, therest -> "${therest}" } + def sha = match.group(1) + def restofline = match.group(2) echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" From b51c69707a3a0de818238981d7a97ac0c830730b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 14:28:43 +0100 Subject: [PATCH 491/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 64 +++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 583c7f1..9e20da2 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -16,13 +16,15 @@ class DifferenceRecord { String rawdifferencetext String sha int type - String jiraticketinfo + String mergeinfo + List jiraticketinfo + String comment DifferenceRecord() { } def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\njiraticketinfo: ${jiraticketinfo}" + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: $(mergeinfo)\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" } } @@ -122,11 +124,45 @@ def extractsha(String line) { def sha = match.group(1) def restofline = match.group(2) - echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" + //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" return [sha.trim(), restofline.trim()] } +def extractmerges(String line) { + def pattern1 = "^(.*)(Merge branch .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + def pattern2 = "^(.*)(Merge remote-tracking branch .*)" + def match2 = (line =~ pattern2) + if (match2.find()) { + return [true, match2.group(2)] + } + + def pattern3 = "^(.*)(Merge commit .*)" + def match3 = (line =~ pattern3) + if (match3.find()) { + return [true, match3.group(2)] + } + + def pattern4 = "^(.*)(Add .* from commit .*)" + def match4 = (line =~ pattern4) + if (match4.find()) { + return [true, match4.group(2)] + } + + return [false, line] +} + +def extractjiraticketinfo(String line) { + bool result = true + + return [result, [], line] +} + def DifferenceRecord fillrecord(meta, line) { def DifferenceRecord dr = new DifferenceRecord() @@ -142,8 +178,26 @@ def DifferenceRecord fillrecord(meta, line) { (sha, line2) = extractsha(line) dr.sha = sha - dr.type = 0 - dr.jiraticketinfo = "info" + (result, mergeinfo) = extractmerges(line2) + if (result) { + dr.type = 1 + dr.jiraticketinfo = [] + dr.mergeinfo = mergeinfo + dr.comment = "" + } else { + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = 2 + dr.jiraticketinfo = jiraticketinfo + dr.mergeinfo = "" + dr.comment = comment + } else { + dr.type = 3 + dr.jiraticketinfo = [] + dr.mergeinfo = "" + df.comment = line2 + } + } return dr } From 299df9b6b219992d7760f9a68524545c9e57b72d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 14:31:16 +0100 Subject: [PATCH 492/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9e20da2..f997983 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -24,7 +24,7 @@ class DifferenceRecord { } def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: $(mergeinfo)\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" } } From 151bbc9599c1d8115d91f1220d0d449ed9c7322d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 18:27:49 +0100 Subject: [PATCH 493/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f997983..5b85543 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -158,7 +158,7 @@ def extractmerges(String line) { } def extractjiraticketinfo(String line) { - bool result = true + boolean result = true return [result, [], line] } From 1c5fae0ab028666c4b083da36ce822211bbfb542 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 19:09:23 +0100 Subject: [PATCH 494/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 47 +++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5b85543..bcc79dc 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -15,7 +15,7 @@ class DifferenceRecord { String rawdifferencetext String sha - int type + String type String mergeinfo List jiraticketinfo String comment @@ -163,6 +163,28 @@ def extractjiraticketinfo(String line) { return [result, [], line] } +def extractreverts(String line) { + boolean result = true + + def pattern1 = '^(.*)(Revert \"(.*)\"' + def match1 = (line =~ pattern1) + if (match1.find()) { + def line2 = match1.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + def pattern2 = '^(.*)(Partial Revert \"(.*)\"' + def match2 = (line =~ pattern2) + if (match2.find()) { + def line2 = match2.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + return [false, [], line] +} + def DifferenceRecord fillrecord(meta, line) { def DifferenceRecord dr = new DifferenceRecord() @@ -180,22 +202,25 @@ def DifferenceRecord fillrecord(meta, line) { (result, mergeinfo) = extractmerges(line2) if (result) { - dr.type = 1 - dr.jiraticketinfo = [] + dr.type = "Merge" dr.mergeinfo = mergeinfo - dr.comment = "" } else { - (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + (result, jiraticketinfo, comment) = extractreverts(line2) if (result) { - dr.type = 2 + dr.type = "Revert" dr.jiraticketinfo = jiraticketinfo - dr.mergeinfo = "" dr.comment = comment } else { - dr.type = 3 - dr.jiraticketinfo = [] - dr.mergeinfo = "" - df.comment = line2 + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.jiraticketinfo = [] + df.comment = line2 + } } } From a2a582cd3988a42fa7e8592079ee8a4f009464c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 20:44:51 +0100 Subject: [PATCH 495/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index bcc79dc..31b2e13 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -166,7 +166,7 @@ def extractjiraticketinfo(String line) { def extractreverts(String line) { boolean result = true - def pattern1 = '^(.*)(Revert \"(.*)\"' + def pattern1 = '^(.*)Revert \"(.*)\"' def match1 = (line =~ pattern1) if (match1.find()) { def line2 = match1.group(2) @@ -174,7 +174,7 @@ def extractreverts(String line) { return [true, originaljiraticketinfo, originalcomment] } - def pattern2 = '^(.*)(Partial Revert \"(.*)\"' + def pattern2 = '^(.*)Partial Revert \"(.*)\"' def match2 = (line =~ pattern2) if (match2.find()) { def line2 = match2.group(2) From c564a70e8de086b626aeaf65f2d8600639b3d354 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 22:08:04 +0100 Subject: [PATCH 496/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 46 ++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 31b2e13..6485c72 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -21,6 +21,16 @@ class DifferenceRecord { String comment DifferenceRecord() { + productname = "" + subcomponentname = "" + jirakey = "" + + rawdifferencetext = "" + sha = "" + type = "" + mergeinfo = "" + jiraticketinfo = [] + comment = "" } def String dump() { @@ -158,9 +168,40 @@ def extractmerges(String line) { } def extractjiraticketinfo(String line) { - boolean result = true + def List jiratickets = [] - return [result, [], line] + boolean amatch = false + while (!amatch) { + line = line.trim() + def pattern1 = '^(\w+-\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def match1 = (line =~ pattern1) + if (match1.find()) { + jiraticket = match1.group(1) + jiratickets.add(jiraticket) + line = match1.group(2) + amatch = true + } else { + def pattern2 = '^(\w+\d+)(.*)' //GIS1234 smb1234 + def match2 = (line =~ pattern2) + if (match2.find()) { + jiraticket = match2.group(1) + jiratickets.add(jiraticket) + line = match2.group(2) + amatch = true + } else { + def pattern3 = '^(\w+ \d+)(.*)' //GIS 1234 smb 1234 + def match3 = (line =~ pattern3) + if (match3.find()) { + jiraticket = match3.group(1) + jiratickets.add(jiraticket) + line = match3.group(2) + amatch = true + } + } + } + } + boolean result = jiratickets.size() > 0 + return [result, jiratickets, line] } def extractreverts(String line) { @@ -218,7 +259,6 @@ def DifferenceRecord fillrecord(meta, line) { dr.comment = comment } else { dr.type = "Unticketed" - dr.jiraticketinfo = [] df.comment = line2 } } From 1f6b1f8aae1032e2fd93c8e16760c1e440e31444 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 22:14:50 +0100 Subject: [PATCH 497/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6485c72..f3ffb80 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -173,7 +173,7 @@ def extractjiraticketinfo(String line) { boolean amatch = false while (!amatch) { line = line.trim() - def pattern1 = '^(\w+-\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) @@ -181,7 +181,7 @@ def extractjiraticketinfo(String line) { line = match1.group(2) amatch = true } else { - def pattern2 = '^(\w+\d+)(.*)' //GIS1234 smb1234 + def pattern2 = '^(\\w+\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) @@ -189,7 +189,7 @@ def extractjiraticketinfo(String line) { line = match2.group(2) amatch = true } else { - def pattern3 = '^(\w+ \d+)(.*)' //GIS 1234 smb 1234 + def pattern3 = '^(\\w+ \\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) From 6c0b5c63c2575104f907fce6a75e3464638fc7dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 4 Jul 2019 23:59:08 +0100 Subject: [PATCH 498/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 65 +++++++++++++++++++++++++++++- differences/metahierarchy.groovy | 4 +- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f3ffb80..cc3d168 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -8,6 +8,8 @@ summarystring = "" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" +reportfolder = "${WORKSPACE}/reports" + class DifferenceRecord { String productname String subcomponentname @@ -45,6 +47,9 @@ def resetfolders() { dir(tempfolder) { deleteDir() } + dir(reportfolder) { + deleteDir() + } } def preconditions() { @@ -326,6 +331,62 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } +def boolean componentreport(meta, componentdifferencerecords) { + def ticketed = [:] + def merges = [:] + def reverts = [:] + def unticketed = [:] + def tickets = [:] + + for (record in componentdifferencerecords) { + if (dr.type == "Merge") { + merges[record.mergeinfo] = record + } + if (dr.type == "Revert") { + for (ticket in record.jiraticketinfo) { + if (!reverts.containsKey(ticket)) { + reverts[ticket] = [record.comment] + } else { + reverts[ticket].add(record.comment) + } + if (!tickets.containsKey(ticket)) { + tickets[ticket] = 1 + } else { + tickets[ticket] += 1 + } + } + } + if (dr.type == "Ticketed") { + for (ticket in record.jiraticketinfo) { + if (!ticketed.containsKey(ticket)) { + ticketed[ticket] = [record.comment] + } else { + ticketed[ticket].add(record.comment) + } + if (!tickets.containsKey(ticket)) { + tickets[ticket] = 1 + } else { + tickets[ticket] += 1 + } + } + } + if (dr.type == "Unticketed") { + unticketed[record.comment] = record + } + } + String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" + dir(componentreportfolder) { + String ticketlist = "Ticket list:\n" + for (ticketnumber in tickets.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key) { + ticketlist += "${ticketnumber} (${tickets[ticketnumber]})" + } + writeFile(file: "ticketlist", text: ticketlist) + } +} +m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key +tickets.sort { a, b -> a.value <=> b.value } + + def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] @@ -346,11 +407,13 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin echo "Later items:" showmetaitems(latermetaitems) - + for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + summaryappend("Items: ${componentdifferencerecords.size()} ") + componentreport(earlymeta, componentdifferencerecords) differencerecords = differencerecords + componentdifferencerecords summaryappend("\n") } else { diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index cd6738d..8a42f5e 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -13,11 +13,13 @@ pipeline { steps { script { SUMMARYDIR = "${WORKSPACE}/summary" + REPORTDIR = "${WORKSPACE}/reports" cleanfolders = [ "${WORKSPACE}/build_artifacts", "${WORKSPACE}/early", "${WORKSPACE}/later", "${SUMMARYDIR}", + "${REPORTDIR}", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -59,7 +61,7 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + archiveArtifacts "summary/**,reports/**,${artifactlist}" } } } From 943ecc2d8d9ea011d2c04a17487d85a735e840da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:02:02 +0100 Subject: [PATCH 499/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index cc3d168..65be091 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -383,8 +383,8 @@ def boolean componentreport(meta, componentdifferencerecords) { writeFile(file: "ticketlist", text: ticketlist) } } -m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key -tickets.sort { a, b -> a.value <=> b.value } +//m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key +//tickets.sort { a, b -> a.value <=> b.value } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { From 0a28daa24fa1ec6db4f6ddf33d98e864524aed65 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:04:38 +0100 Subject: [PATCH 500/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 65be091..d5f0fdb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -338,16 +338,16 @@ def boolean componentreport(meta, componentdifferencerecords) { def unticketed = [:] def tickets = [:] - for (record in componentdifferencerecords) { + for (dr in componentdifferencerecords) { if (dr.type == "Merge") { - merges[record.mergeinfo] = record + merges[dr.mergeinfo] = dr } if (dr.type == "Revert") { - for (ticket in record.jiraticketinfo) { + for (ticket in dr.jiraticketinfo) { if (!reverts.containsKey(ticket)) { - reverts[ticket] = [record.comment] + reverts[ticket] = [dr.comment] } else { - reverts[ticket].add(record.comment) + reverts[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 @@ -357,11 +357,11 @@ def boolean componentreport(meta, componentdifferencerecords) { } } if (dr.type == "Ticketed") { - for (ticket in record.jiraticketinfo) { + for (ticket in dr.jiraticketinfo) { if (!ticketed.containsKey(ticket)) { - ticketed[ticket] = [record.comment] + ticketed[ticket] = [dr.comment] } else { - ticketed[ticket].add(record.comment) + ticketed[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 @@ -371,7 +371,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } } if (dr.type == "Unticketed") { - unticketed[record.comment] = record + unticketed[dr.comment] = dr } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" From 99238fb9c521c89efeda2b2a0b0dd8fb4025f452 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:09:36 +0100 Subject: [PATCH 501/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index d5f0fdb..30a0442 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,7 +377,7 @@ def boolean componentreport(meta, componentdifferencerecords) { String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" dir(componentreportfolder) { String ticketlist = "Ticket list:\n" - for (ticketnumber in tickets.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key) { + for (ticketnumber in tickets.sort()*.key) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})" } writeFile(file: "ticketlist", text: ticketlist) @@ -385,7 +385,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key //tickets.sort { a, b -> a.value <=> b.value } - +//myMap.sort()*.key def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] From cfff9bd1025f9786ec6d3e580086f30de612785f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:32:12 +0100 Subject: [PATCH 502/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 45 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 30a0442..effa08f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -306,21 +306,6 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { return [result, differencerecords] } -def compare(earlymetaitems, latermetaitems, key) { - boolean result = true; - echo "Collecting changes between early and later versions of component: ${key}" - summaryappend("Changed: ") - - def earlymeta = earlymetaitems[key].meta - def latermeta = latermetaitems[key].meta - - def uniquefolderforsourcerepo = getcomponent(earlymeta) - - (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - - return [result, differencerecords]; -} - def deletedcomponent(earlymetaitems, key) { echo "Component ${key} has been deleted in later version" summaryappend("Deleted ") @@ -378,7 +363,7 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" for (ticketnumber in tickets.sort()*.key) { - ticketlist += "${ticketnumber} (${tickets[ticketnumber]})" + ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } writeFile(file: "ticketlist", text: ticketlist) } @@ -387,6 +372,26 @@ def boolean componentreport(meta, componentdifferencerecords) { //tickets.sort { a, b -> a.value <=> b.value } //myMap.sort()*.key +def compare(earlymetaitems, latermetaitems, key) { + boolean result = true; + echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].meta + def latermeta = latermetaitems[key].meta + + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + + def uniquefolderforsourcerepo = getcomponent(earlymeta) + + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + summaryappend(" Items: ${differencerecords.size()} ") + componentreport(earlymeta, differencerecords) + summaryappend("\n") + + return [result, differencerecords]; +} + def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] @@ -410,12 +415,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) - summaryappend("Items: ${componentdifferencerecords.size()} ") - componentreport(earlymeta, componentdifferencerecords) differencerecords = differencerecords + componentdifferencerecords - summaryappend("\n") } else { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) @@ -435,4 +436,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin return [summarystring, markunstable, unstablereason] } + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + + return this; From 7e3042e0d27b6334e590ec904a26b2eded64ac53 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 00:34:35 +0100 Subject: [PATCH 503/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index effa08f..f8650be 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -436,8 +436,4 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin return [summarystring, markunstable, unstablereason] } - for (key in metaitems.keySet()) { - def relatedmetainfo = metaitems[key] - - return this; From a260f34a1525cf6dd485af62e9c0be620222b203 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 08:21:49 +0100 Subject: [PATCH 504/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f8650be..5734bb6 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -324,38 +324,52 @@ def boolean componentreport(meta, componentdifferencerecords) { def tickets = [:] for (dr in componentdifferencerecords) { + echo "Processing:\n${dr.dump()}" + if (dr.type == "Merge") { + echo "Adding to mergeinfo" merges[dr.mergeinfo] = dr } if (dr.type == "Revert") { + echo "Adding to revert" for (ticket in dr.jiraticketinfo) { if (!reverts.containsKey(ticket)) { + echo "Creating reverts record for ${ticket}" reverts[ticket] = [dr.comment] } else { + echo "Adding to ticketed record for ${ticket}" reverts[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { + echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { + echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } } } if (dr.type == "Ticketed") { + echo "Adding to ticketed" for (ticket in dr.jiraticketinfo) { if (!ticketed.containsKey(ticket)) { + echo "Creating ticketed record for ${ticket}" ticketed[ticket] = [dr.comment] } else { + echo "Adding to ticketed record for ${ticket}" ticketed[ticket].add(dr.comment) } if (!tickets.containsKey(ticket)) { + echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { + echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } } } if (dr.type == "Unticketed") { + echo "Adding to unticketed" unticketed[dr.comment] = dr } } @@ -430,9 +444,6 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin summaryappend("\n") } } - for (record in differencerecords) { - echo "${record.dump()}" - } return [summarystring, markunstable, unstablereason] } From 97faaf812e28ef0328d0d2863c0640d0cb3798f5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 08:31:05 +0100 Subject: [PATCH 505/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5734bb6..1b622e0 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -347,6 +347,7 @@ def boolean componentreport(meta, componentdifferencerecords) { echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } + echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Ticketed") { @@ -366,6 +367,7 @@ def boolean componentreport(meta, componentdifferencerecords) { echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } + echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Unticketed") { @@ -374,6 +376,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" + echo "Size of tickets map: ${tickets.size()}" dir(componentreportfolder) { String ticketlist = "Ticket list:\n" for (ticketnumber in tickets.sort()*.key) { From 9c2d045a1883be3a3dc5ac1687f5a1d789a763c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 08:36:05 +0100 Subject: [PATCH 506/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1b622e0..861dbca 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -379,6 +379,8 @@ def boolean componentreport(meta, componentdifferencerecords) { echo "Size of tickets map: ${tickets.size()}" dir(componentreportfolder) { String ticketlist = "Ticket list:\n" + echo "tickets map:\n${tickets}" + echo "tickets map sorted:\n${tickets.sort()*.key}" for (ticketnumber in tickets.sort()*.key) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } From 3936752cb5e47fadd0201dfa16abc85611af7fa6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 09:14:46 +0100 Subject: [PATCH 507/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 861dbca..1a4cb36 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -380,8 +380,9 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - echo "tickets map sorted:\n${tickets.sort()*.key}" - for (ticketnumber in tickets.sort()*.key) { + tickets = tickets.sort { it.key } + echo "tickets map sorted:\n${tickets}" + for (ticketnumber in tickets) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } writeFile(file: "ticketlist", text: ticketlist) From d5b25aa338c5895918489753256d015d9e93db15 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 09:34:15 +0100 Subject: [PATCH 508/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1a4cb36..a21d475 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -91,9 +91,6 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { git reset --hard git clean -fdx git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null """ ) } @@ -380,9 +377,9 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - tickets = tickets.sort { it.key } - echo "tickets map sorted:\n${tickets}" - for (ticketnumber in tickets) { + ticketssorted = tickets.sort { it.key } + echo "tickets map sorted:\n${ticketssorted}" + for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" } writeFile(file: "ticketlist", text: ticketlist) From 79abd7ba56717effb371d2ba7aec72c472a599cb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 11:56:56 +0100 Subject: [PATCH 509/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a21d475..6df7368 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,7 +377,7 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - ticketssorted = tickets.sort { it.key } + ticketssorted = tickets.toSorted { a, b -> a.key <=> b.key } echo "tickets map sorted:\n${ticketssorted}" for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" From ae6c895c8ad3c7332d7fc0c47d6bfdacbaf52376 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 12:48:02 +0100 Subject: [PATCH 510/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6df7368..cb23c87 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -314,11 +314,11 @@ def addedcomponent(latermetaitems, key) { } def boolean componentreport(meta, componentdifferencerecords) { - def ticketed = [:] - def merges = [:] - def reverts = [:] - def unticketed = [:] - def tickets = [:] + def Map ticketed = [:] + def Map merges = [:] + def Map reverts = [:] + def Map unticketed = [:] + def Map tickets = [:] for (dr in componentdifferencerecords) { echo "Processing:\n${dr.dump()}" @@ -377,7 +377,7 @@ def boolean componentreport(meta, componentdifferencerecords) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" echo "tickets map:\n${tickets}" - ticketssorted = tickets.toSorted { a, b -> a.key <=> b.key } + ticketssorted = tickets.sort() echo "tickets map sorted:\n${ticketssorted}" for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" From 7a94c993c6f090cbaab143b117a3b900b5174c46 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:01:10 +0100 Subject: [PATCH 511/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 40 ++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index cb23c87..043a0ca 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,6 +313,32 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } +def componentmergereport(String componentreportfolder, merges) { + echo "Size of merges map: ${merges.size()}" + dir(componentreportfolder) { + String mergelist = "Merge list:\n" + echo "merges map:\n${merges}" + mergessorted = merges.sort() + echo "merges map sorted:\n${mergessorted}" + for (amerge in mergessorted) { + mergelist += "${amerge.key}\n" + } + writeFile(file: "mergelist.txt", text: mergelist) + } +} + +def componentticketsreport(String componentreportfolder, tickets) { + echo "Size of tickets map: ${tickets.size()}" + dir(componentreportfolder) { + String ticketlist = "Ticket list:\n" + ticketssorted = tickets.sort() + for (ticketnumber in ticketssorted) { + ticketlist += "${ticketnumber.key} (${tickets.value})\n" + } + writeFile(file: "ticketlist.txt", text: ticketlist) + } +} + def boolean componentreport(meta, componentdifferencerecords) { def Map ticketed = [:] def Map merges = [:] @@ -373,17 +399,9 @@ def boolean componentreport(meta, componentdifferencerecords) { } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" - echo "Size of tickets map: ${tickets.size()}" - dir(componentreportfolder) { - String ticketlist = "Ticket list:\n" - echo "tickets map:\n${tickets}" - ticketssorted = tickets.sort() - echo "tickets map sorted:\n${ticketssorted}" - for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber} (${tickets[ticketnumber]})\n" - } - writeFile(file: "ticketlist", text: ticketlist) - } + componentmergereport(componentreportfolder, merges) + componentticketreport(componentreportfolder, tickets) + } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key //tickets.sort { a, b -> a.value <=> b.value } From e982b587219d6958f933c1542f307ad3310e2ecd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:04:38 +0100 Subject: [PATCH 512/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 043a0ca..3d08eb8 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -400,7 +400,7 @@ def boolean componentreport(meta, componentdifferencerecords) { } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" componentmergereport(componentreportfolder, merges) - componentticketreport(componentreportfolder, tickets) + componentticketsreport(componentreportfolder, tickets) } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key From fdec3b5601069aae577252beb20da369cdc2ce89 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:16:56 +0100 Subject: [PATCH 513/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3d08eb8..a384a07 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,13 +313,10 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def componentmergereport(String componentreportfolder, merges) { - echo "Size of merges map: ${merges.size()}" +def mergereport(String componentreportfolder, merges) { dir(componentreportfolder) { String mergelist = "Merge list:\n" - echo "merges map:\n${merges}" mergessorted = merges.sort() - echo "merges map sorted:\n${mergessorted}" for (amerge in mergessorted) { mergelist += "${amerge.key}\n" } @@ -327,13 +324,26 @@ def componentmergereport(String componentreportfolder, merges) { } } -def componentticketsreport(String componentreportfolder, tickets) { - echo "Size of tickets map: ${tickets.size()}" +def revertsreport(String componentreportfolder, reverts) { + echo "Size of reverts map: ${reverts.size()}" + dir(componentreportfolder) { + String revertlist = "Revert list:\n" + echo "reverts map:\n${reverts}" + revertssorted = reverts.sort() + echo "reverts map sorted:\n${revertssorted}" + for (arevert in revertssorted) { + revertlist += "${arevert.key}:\n${arevert.value.join("\n")}\n" + } + writeFile(file: "revertlist.txt", text: revertlist) + } +} + +def ticketsreport(String componentreportfolder, tickets) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber.key} (${tickets.value})\n" + ticketlist += "${ticketnumber.key} (${ticketnumber.value})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } @@ -399,8 +409,9 @@ def boolean componentreport(meta, componentdifferencerecords) { } } String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" - componentmergereport(componentreportfolder, merges) - componentticketsreport(componentreportfolder, tickets) + mergereport(componentreportfolder, merges) + revertsreport(componentreportfolder, reverts) + ticketsreport(componentreportfolder, tickets) } //m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key From 248803207feae3e8daecf8838d3e1aa8f5691b0c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:23:08 +0100 Subject: [PATCH 514/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a384a07..4c1bffe 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -338,6 +338,20 @@ def revertsreport(String componentreportfolder, reverts) { } } +def ticketedreport(String componentreportfolder, ticketed) { + echo "Size of ticketed map: ${ticketed.size()}" + dir(componentreportfolder) { + String ticketedlist = "Ticketed list:\n" + echo "ticketed map:\n${ticketed}" + ticketedsorted = ticketed.sort() + echo "ticketed map sorted:\n${ticketedsorted}" + for (aticketed in ticketedsorted) { + ticketedlist += "${aticketed.key}:\n${aticketed.value.join("\n")}\n" + } + writeFile(file: "ticketedlist.txt", text: ticketedlist) + } +} + def ticketsreport(String componentreportfolder, tickets) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" @@ -411,6 +425,7 @@ def boolean componentreport(meta, componentdifferencerecords) { String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" mergereport(componentreportfolder, merges) revertsreport(componentreportfolder, reverts) + ticketedreport(String componentreportfolder, ticketed) ticketsreport(componentreportfolder, tickets) } From 807ba4492dcdd3e4bed8f1dd9dd2778849ffb47d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:32:51 +0100 Subject: [PATCH 515/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 4c1bffe..fef482b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -425,7 +425,7 @@ def boolean componentreport(meta, componentdifferencerecords) { String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" mergereport(componentreportfolder, merges) revertsreport(componentreportfolder, reverts) - ticketedreport(String componentreportfolder, ticketed) + ticketedreport(componentreportfolder, ticketed) ticketsreport(componentreportfolder, tickets) } From 437a15986dd6d3f1cc3c7a7cb6ceca595b144502 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 13:52:26 +0100 Subject: [PATCH 516/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 55 ++++++++++++++++++------------ 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index fef482b..29c8a43 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,7 +313,7 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def mergereport(String componentreportfolder, merges) { +def mergereport(String componentreportfolder, merges, boolean iscomponent = true) { dir(componentreportfolder) { String mergelist = "Merge list:\n" mergessorted = merges.sort() @@ -324,13 +324,10 @@ def mergereport(String componentreportfolder, merges) { } } -def revertsreport(String componentreportfolder, reverts) { - echo "Size of reverts map: ${reverts.size()}" +def revertsreport(String componentreportfolder, reverts, boolean iscomponent = true) { dir(componentreportfolder) { String revertlist = "Revert list:\n" - echo "reverts map:\n${reverts}" revertssorted = reverts.sort() - echo "reverts map sorted:\n${revertssorted}" for (arevert in revertssorted) { revertlist += "${arevert.key}:\n${arevert.value.join("\n")}\n" } @@ -338,13 +335,10 @@ def revertsreport(String componentreportfolder, reverts) { } } -def ticketedreport(String componentreportfolder, ticketed) { - echo "Size of ticketed map: ${ticketed.size()}" +def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = true) { dir(componentreportfolder) { String ticketedlist = "Ticketed list:\n" - echo "ticketed map:\n${ticketed}" ticketedsorted = ticketed.sort() - echo "ticketed map sorted:\n${ticketedsorted}" for (aticketed in ticketedsorted) { ticketedlist += "${aticketed.key}:\n${aticketed.value.join("\n")}\n" } @@ -352,7 +346,21 @@ def ticketedreport(String componentreportfolder, ticketed) { } } -def ticketsreport(String componentreportfolder, tickets) { +def unticketedreport(String componentreportfolder, unticketed, boolean iscomponent = true) { + echo "Size of ticketed map: ${unticketed.size()}" + dir(componentreportfolder) { + String unticketedlist = "Unticketed list:\n" + echo "unticketed map:\n${unticketed}" + unticketedsorted = unticketed.sort() + echo "unticketed map sorted:\n${unticketedsorted}" + for (anunticketed in unticketedsorted) { + unticketedlist += "${anunticketed.key}\n" + } + writeFile(file: "unticketedlist.txt", text: unticketedlist) + } +} + +def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = true) { dir(componentreportfolder) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.sort() @@ -363,14 +371,14 @@ def ticketsreport(String componentreportfolder, tickets) { } } -def boolean componentreport(meta, componentdifferencerecords) { +def boolean report(meta, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] def Map merges = [:] def Map reverts = [:] def Map unticketed = [:] def Map tickets = [:] - for (dr in componentdifferencerecords) { + for (dr in differencerecords) { echo "Processing:\n${dr.dump()}" if (dr.type == "Merge") { @@ -422,16 +430,18 @@ def boolean componentreport(meta, componentdifferencerecords) { unticketed[dr.comment] = dr } } - String componentreportfolder = "${reportfolder}/${meta.componentidentifier()}" - mergereport(componentreportfolder, merges) - revertsreport(componentreportfolder, reverts) - ticketedreport(componentreportfolder, ticketed) - ticketsreport(componentreportfolder, tickets) - + String reportsubfolder = "" + if (iscomponent) { + reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" + } else { + reportsubfolder = "${reportfolder}/Combined" + } + mergereport(reportsubfolder, merges, iscomponent) + revertsreport(reportsubfolder, reverts, iscomponent) + ticketedreport(reportsubfolder, ticketed, iscomponent) + unticketedreport(reportsubfolder, unticketed, iscomponent) + ticketsreport(reportsubfolder, tickets, iscomponent) } -//m.sort( { k1, k2 -> k1 <=> k2 } as Comparator )*.key -//tickets.sort { a, b -> a.value <=> b.value } -//myMap.sort()*.key def compare(earlymetaitems, latermetaitems, key) { boolean result = true; @@ -447,7 +457,7 @@ def compare(earlymetaitems, latermetaitems, key) { (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) summaryappend(" Items: ${differencerecords.size()} ") - componentreport(earlymeta, differencerecords) + report(earlymeta, differencerecords) summaryappend("\n") return [result, differencerecords]; @@ -491,6 +501,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin summaryappend("\n") } } + report(meta, differencerecords, false) return [summarystring, markunstable, unstablereason] } From 5122aca1bf70a5b54137f21b000a2f3300bb5918 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 14:34:32 +0100 Subject: [PATCH 517/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 29c8a43..dc174a7 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -329,7 +329,10 @@ def revertsreport(String componentreportfolder, reverts, boolean iscomponent = t String revertlist = "Revert list:\n" revertssorted = reverts.sort() for (arevert in revertssorted) { - revertlist += "${arevert.key}:\n${arevert.value.join("\n")}\n" + revertlist += "${arevert.key}:\n" + for (revertlistcomment in arevert.value) { + revertlist += "${revertlistcomment.key} (${revertlistcomment.value})\n" + } } writeFile(file: "revertlist.txt", text: revertlist) } @@ -340,7 +343,10 @@ def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { - ticketedlist += "${aticketed.key}:\n${aticketed.value.join("\n")}\n" + ticketedlist += "${aticketed.key}:\n" + for (ticketedlistcomment in aticketed.value) { + ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" + } } writeFile(file: "ticketedlist.txt", text: ticketedlist) } @@ -355,6 +361,9 @@ def unticketedreport(String componentreportfolder, unticketed, boolean iscompone echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { unticketedlist += "${anunticketed.key}\n" + for (unticketedlistcomment in anunticketed.value) { + unticketedlist += "${unticketedlistcomment.key} (${unticketedlistcomment.value})\n" + } } writeFile(file: "unticketedlist.txt", text: unticketedlist) } @@ -390,10 +399,11 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (ticket in dr.jiraticketinfo) { if (!reverts.containsKey(ticket)) { echo "Creating reverts record for ${ticket}" - reverts[ticket] = [dr.comment] + reverts[ticket] = [:] + reverts[ticket][dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - reverts[ticket].add(dr.comment) + reverts[ticket][dr.comment] += 1 } if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" @@ -410,10 +420,11 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (ticket in dr.jiraticketinfo) { if (!ticketed.containsKey(ticket)) { echo "Creating ticketed record for ${ticket}" - ticketed[ticket] = [dr.comment] + ticketed[ticket] = [:] + ticketed[ticket][dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - ticketed[ticket].add(dr.comment) + ticketed[ticket][dr.comment] += 1 } if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" From 83e50fc58518da467ed89cc663fa85029854cc2c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:01:28 +0100 Subject: [PATCH 518/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dc174a7..c337e9a 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -397,14 +397,18 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (dr.type == "Revert") { echo "Adding to revert" for (ticket in dr.jiraticketinfo) { + def comments = [:] if (!reverts.containsKey(ticket)) { echo "Creating reverts record for ${ticket}" reverts[ticket] = [:] - reverts[ticket][dr.comment] = 1 + comments = reverts[ticket] + comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - reverts[ticket][dr.comment] += 1 + comments = reverts[ticket] + comments[dr.comment] += 1 } + reverts[ticket] = comments if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 @@ -418,14 +422,18 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (dr.type == "Ticketed") { echo "Adding to ticketed" for (ticket in dr.jiraticketinfo) { + def comments = [:] if (!ticketed.containsKey(ticket)) { echo "Creating ticketed record for ${ticket}" ticketed[ticket] = [:] - ticketed[ticket][dr.comment] = 1 + comments = ticketed[ticket] + comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - ticketed[ticket][dr.comment] += 1 + def comments = ticketed[ticket] + comments[dr.comment] += 1 } + ticketed[ticket] = comments if (!tickets.containsKey(ticket)) { echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 From 11fcc3279bf20505d00a7412813f9ec0cd8b51f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:03:20 +0100 Subject: [PATCH 519/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index c337e9a..c1449f5 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -430,7 +430,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - def comments = ticketed[ticket] + comments = ticketed[ticket] comments[dr.comment] += 1 } ticketed[ticket] = comments From 2f1a51c7582e9ebd417831abc92a1aeef9f09dee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:34:57 +0100 Subject: [PATCH 520/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index c1449f5..dfb75c4 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -405,8 +405,12 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments[dr.comment] = 1 } else { echo "Adding to ticketed record for ${ticket}" - comments = reverts[ticket] - comments[dr.comment] += 1 + comments = ticketed[ticket] + if (!comments.containsKey(dr.comment)) { + comments[dr.comment] = 1 + } else { + comments[dr.comment] += 1 + } } reverts[ticket] = comments if (!tickets.containsKey(ticket)) { @@ -431,7 +435,11 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } else { echo "Adding to ticketed record for ${ticket}" comments = ticketed[ticket] - comments[dr.comment] += 1 + if (!comments.containsKey(dr.comment)) { + comments[dr.comment] = 1 + } else { + comments[dr.comment] += 1 + } } ticketed[ticket] = comments if (!tickets.containsKey(ticket)) { From b3b47d259328d157433d8b7cd3d6453322d4a447 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:45:00 +0100 Subject: [PATCH 521/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dfb75c4..cd988ac 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -380,6 +380,16 @@ def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = t } } +def differencerecordsreport(String reportsubfolder, differencerecords, , boolean iscomponent = true) { + dir(componentreportfolder) { + String recordlist = "" + for (record in differencerecords) { + recordlist += dr.dump() + } + writeFile(file: "recordlist.txt", text: recordlist) + } +} + def boolean report(meta, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] def Map merges = [:] @@ -388,8 +398,6 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { def Map tickets = [:] for (dr in differencerecords) { - echo "Processing:\n${dr.dump()}" - if (dr.type == "Merge") { echo "Adding to mergeinfo" merges[dr.mergeinfo] = dr @@ -468,6 +476,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { ticketedreport(reportsubfolder, ticketed, iscomponent) unticketedreport(reportsubfolder, unticketed, iscomponent) ticketsreport(reportsubfolder, tickets, iscomponent) + differencerecordsreport(reportsubfolder, differencerecords, iscomponent) } def compare(earlymetaitems, latermetaitems, key) { From 095ae84b697ad192ccd24ba54dbea017334cdc6f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 15:48:19 +0100 Subject: [PATCH 522/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index cd988ac..203d09b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -380,7 +380,7 @@ def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = t } } -def differencerecordsreport(String reportsubfolder, differencerecords, , boolean iscomponent = true) { +def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { dir(componentreportfolder) { String recordlist = "" for (record in differencerecords) { From e441a724ce95b0be3cc8c866258984992ea55ee2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 16:01:18 +0100 Subject: [PATCH 523/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 203d09b..e1d684b 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -313,8 +313,8 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def mergereport(String componentreportfolder, merges, boolean iscomponent = true) { - dir(componentreportfolder) { +def mergereport(String reportsubfolder, merges, boolean iscomponent = true) { + dir(reportsubfolder) { String mergelist = "Merge list:\n" mergessorted = merges.sort() for (amerge in mergessorted) { @@ -324,8 +324,8 @@ def mergereport(String componentreportfolder, merges, boolean iscomponent = true } } -def revertsreport(String componentreportfolder, reverts, boolean iscomponent = true) { - dir(componentreportfolder) { +def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { + dir(reportsubfolder) { String revertlist = "Revert list:\n" revertssorted = reverts.sort() for (arevert in revertssorted) { @@ -338,8 +338,8 @@ def revertsreport(String componentreportfolder, reverts, boolean iscomponent = t } } -def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = true) { - dir(componentreportfolder) { +def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { + dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { @@ -352,9 +352,9 @@ def ticketedreport(String componentreportfolder, ticketed, boolean iscomponent = } } -def unticketedreport(String componentreportfolder, unticketed, boolean iscomponent = true) { +def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { echo "Size of ticketed map: ${unticketed.size()}" - dir(componentreportfolder) { + dir(reportsubfolder) { String unticketedlist = "Unticketed list:\n" echo "unticketed map:\n${unticketed}" unticketedsorted = unticketed.sort() @@ -369,8 +369,8 @@ def unticketedreport(String componentreportfolder, unticketed, boolean iscompone } } -def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = true) { - dir(componentreportfolder) { +def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { + dir(reportsubfolder) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { @@ -381,7 +381,7 @@ def ticketsreport(String componentreportfolder, tickets, boolean iscomponent = t } def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { - dir(componentreportfolder) { + dir(reportsubfolder) { String recordlist = "" for (record in differencerecords) { recordlist += dr.dump() From e95197837f6a5660cbc7be2f6d6a3eef8b76f3ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 16:05:14 +0100 Subject: [PATCH 524/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e1d684b..34b90ec 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -383,7 +383,7 @@ def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { dir(reportsubfolder) { String recordlist = "" - for (record in differencerecords) { + for (dr in differencerecords) { recordlist += dr.dump() } writeFile(file: "recordlist.txt", text: recordlist) @@ -399,20 +399,16 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (dr in differencerecords) { if (dr.type == "Merge") { - echo "Adding to mergeinfo" merges[dr.mergeinfo] = dr } if (dr.type == "Revert") { - echo "Adding to revert" for (ticket in dr.jiraticketinfo) { def comments = [:] if (!reverts.containsKey(ticket)) { - echo "Creating reverts record for ${ticket}" reverts[ticket] = [:] comments = reverts[ticket] comments[dr.comment] = 1 } else { - echo "Adding to ticketed record for ${ticket}" comments = ticketed[ticket] if (!comments.containsKey(dr.comment)) { comments[dr.comment] = 1 @@ -422,26 +418,20 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } reverts[ticket] = comments if (!tickets.containsKey(ticket)) { - echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { - echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } - echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Ticketed") { - echo "Adding to ticketed" for (ticket in dr.jiraticketinfo) { def comments = [:] if (!ticketed.containsKey(ticket)) { - echo "Creating ticketed record for ${ticket}" ticketed[ticket] = [:] comments = ticketed[ticket] comments[dr.comment] = 1 } else { - echo "Adding to ticketed record for ${ticket}" comments = ticketed[ticket] if (!comments.containsKey(dr.comment)) { comments[dr.comment] = 1 @@ -451,17 +441,13 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } ticketed[ticket] = comments if (!tickets.containsKey(ticket)) { - echo "Creating ticket record for ${ticket}" tickets[ticket] = 1 } else { - echo "Incrementing ticket record for ${ticket}" tickets[ticket] += 1 } - echo "tickets size now: ${tickets.size()}" } } if (dr.type == "Unticketed") { - echo "Adding to unticketed" unticketed[dr.comment] = dr } } From 15cf1e840a122536023ee50a9feebf45336309b9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:11:52 +0100 Subject: [PATCH 525/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- differences/metahierarchy.groovy | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 34b90ec..26230ab 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -384,7 +384,7 @@ def differencerecordsreport(String reportsubfolder, differencerecords, boolean i dir(reportsubfolder) { String recordlist = "" for (dr in differencerecords) { - recordlist += dr.dump() + recordlist += dr.dump() + "\n" } writeFile(file: "recordlist.txt", text: recordlist) } diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 8a42f5e..3392fdc 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -14,10 +14,15 @@ pipeline { script { SUMMARYDIR = "${WORKSPACE}/summary" REPORTDIR = "${WORKSPACE}/reports" + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + cleanfolders = [ "${WORKSPACE}/build_artifacts", - "${WORKSPACE}/early", - "${WORKSPACE}/later", + "${EARLYDIR}", + "${LATERDIR}", "${SUMMARYDIR}", "${REPORTDIR}", ] @@ -26,22 +31,14 @@ pipeline { } } } - stage('Checkout top level meta') { + stage('Obtain top level meta') { steps { script { - configurationhelper = autobuilder.configurationhelper() + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - - EARLYREVISION = RELEASE_EARLY_REVISION.trim() - EARLYROOTDIR = 'early' - EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" - configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) - - LATERREVISION = RELEASE_LATER_REVISION.trim() - LATERROOTDIR = 'later' - LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" - configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) + importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR) + importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR) } } } From 5520e3d09708a6c3b60223e4184aa4f296308a6e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:33:32 +0100 Subject: [PATCH 526/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 26230ab..9db7309 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -123,7 +123,7 @@ def getgitlogdifferences(String gitrootfolder, String earlyrevision, String late } def String differencetext dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git log --oneline ${releaseexpression}").trim() + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() } return differencetext } From 67a3458e96ed9584ff71c6b14a6360c148fdde9a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:40:04 +0100 Subject: [PATCH 527/589] Add more difference infrastructure --- differences/metahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 3392fdc..4bcdcdc 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -37,8 +37,8 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR) - importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR) + importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') + importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') } } } From 529067c8442a0610f765f3ac3302c8f04eedfb2f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:47:11 +0100 Subject: [PATCH 528/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9db7309..46a2943 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -275,6 +275,7 @@ def List processtext(meta, String text) { text.split('\n').each { DifferenceRecord dr = fillrecord(meta, it.trim()) differencerecords.add(dr) + echo "${dr.dump()}" } return differencerecords } From 1743fdef3dae5ff73771dea12b9ce7e7664f7e68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 17:50:16 +0100 Subject: [PATCH 529/589] Add more difference infrastructure --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 7e065d5..3ea97ec 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -251,7 +251,7 @@ def List convertSparseItemsString(String itemstopublish) { boolean nonempty = itemstopublish?.trim() if (nonempty) { - def String[] items = itemstopublish..trim().split(',') + def String[] items = itemstopublish.trim().split(',') def String allpaths = "" for(String item : items) { sparsepaths.add(item) From a95eb0c1a251d9d595438d17c593c6fe6e6fa795 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 18:06:21 +0100 Subject: [PATCH 530/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 46a2943..67b08f3 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -172,8 +172,7 @@ def extractmerges(String line) { def extractjiraticketinfo(String line) { def List jiratickets = [] - boolean amatch = false - while (!amatch) { + while (true) { line = line.trim() def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) @@ -181,7 +180,6 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) jiratickets.add(jiraticket) line = match1.group(2) - amatch = true } else { def pattern2 = '^(\\w+\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) @@ -189,7 +187,6 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) jiratickets.add(jiraticket) line = match2.group(2) - amatch = true } else { def pattern3 = '^(\\w+ \\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) @@ -197,8 +194,9 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) jiratickets.add(jiraticket) line = match3.group(2) - amatch = true } + } else { + break } } } From 81717b573f8e8e6e1b01de92d1b74d87e0f414a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 18:14:51 +0100 Subject: [PATCH 531/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 67b08f3..f98ee06 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -194,10 +194,10 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) jiratickets.add(jiraticket) line = match3.group(2) + } else { + break } - } else { - break - } + } } } boolean result = jiratickets.size() > 0 From e7bf289130db82cca79ac19992189e14519604b4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 5 Jul 2019 18:17:54 +0100 Subject: [PATCH 532/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f98ee06..1b6055e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -259,7 +259,7 @@ def DifferenceRecord fillrecord(meta, line) { dr.comment = comment } else { dr.type = "Unticketed" - df.comment = line2 + dr.comment = line2 } } } From 29b0981c8a7a07e81e646193a633dce9d14c4965 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 10:55:38 +0100 Subject: [PATCH 533/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1b6055e..0a71913 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -359,10 +359,7 @@ def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = t unticketedsorted = unticketed.sort() echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key}\n" - for (unticketedlistcomment in anunticketed.value) { - unticketedlist += "${unticketedlistcomment.key} (${unticketedlistcomment.value})\n" - } + unticketedlist += "${anunticketed.key}\n${anunticketed.value()}" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } @@ -447,7 +444,12 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } } if (dr.type == "Unticketed") { - unticketed[dr.comment] = dr + if (!unticketed.containsKey(dr.comment)) { + unticketed[dr.comment] = 1 + } else { + unticketed[dr.comment] += 1 + } + } } } String reportsubfolder = "" From e8122dbab1f3eff143d55504e5d4762dc24b02de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 10:58:56 +0100 Subject: [PATCH 534/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 0a71913..ca2a633 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -446,9 +446,8 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (dr.type == "Unticketed") { if (!unticketed.containsKey(dr.comment)) { unticketed[dr.comment] = 1 - } else { - unticketed[dr.comment] += 1 - } + } else { + unticketed[dr.comment] += 1 } } } From e98a769dd1b0c29812e53c7c19e9988d602df2c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 11:02:20 +0100 Subject: [PATCH 535/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index ca2a633..379c482 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -359,7 +359,7 @@ def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = t unticketedsorted = unticketed.sort() echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key}\n${anunticketed.value()}" + unticketedlist += "${anunticketed.key}\n${anunticketed.value}" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } From 0280c4ec003d8c004382fb52370beaf6b5cef5e1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 11:14:16 +0100 Subject: [PATCH 536/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 379c482..f894e65 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -405,7 +405,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments = reverts[ticket] comments[dr.comment] = 1 } else { - comments = ticketed[ticket] + comments = reverts[ticket] if (!comments.containsKey(dr.comment)) { comments[dr.comment] = 1 } else { From 750955d8cdffc153bde3a6a4871ed3ac858b34a7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 12:26:55 +0100 Subject: [PATCH 537/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 57 ++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index f894e65..c29a278 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -169,6 +169,21 @@ def extractmerges(String line) { return [false, line] } +def jiraticketexists(String jiraticket) { + def String URI = "${JIRA_URI}/browse/${jiraticket}" + def get = new URL(URI).openConnection(); + get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) + get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) + get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) + get.setRequestProperty( 'Accept', 'application/json' ) + def getRC = get.getResponseCode(); + + if(!getRC.equals(200)) { + return [false, ''] + } + return [true, URI] +} + def extractjiraticketinfo(String line) { def List jiratickets = [] @@ -178,22 +193,37 @@ def extractjiraticketinfo(String line) { def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) - jiratickets.add(jiraticket) - line = match1.group(2) - } else { - def pattern2 = '^(\\w+\\d+)(.*)' //GIS1234 smb1234 + (exists, URI) = jiraticketexists(jiraticket) + if (exists) { + jiratickets.add([jiraticket, URI]) + line = match1.group(2) + } else { + break + } + } else { + def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) if (match2.find()) { - jiraticket = match2.group(1) - jiratickets.add(jiraticket) - line = match2.group(2) + jiraticket = match2.group(1) + "-" + match2.group(2) + (exists, URI) = jiraticketexists(jiraticket) + if (exists) { + jiratickets.add([jiraticket, URI]) + line = match2.group(3) + } else { + break + } } else { - def pattern3 = '^(\\w+ \\d+)(.*)' //GIS 1234 smb 1234 + def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) if (match3.find()) { - jiraticket = match3.group(1) - jiratickets.add(jiraticket) - line = match3.group(2) + jiraticket = match3.group(1) + "-" + match3.group(2) + (exists, URI) = jiraticketexists(jiraticket) + if (exists) { + jiratickets.add([jiraticket, URI]) + line = match3.group(3) + } else { + break + } } else { break } @@ -352,14 +382,11 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) } def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { - echo "Size of ticketed map: ${unticketed.size()}" dir(reportsubfolder) { String unticketedlist = "Unticketed list:\n" - echo "unticketed map:\n${unticketed}" unticketedsorted = unticketed.sort() - echo "unticketed map sorted:\n${unticketedsorted}" for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key}\n${anunticketed.value}" + unticketedlist += "${anunticketed.key} (${anunticketed.value})\n" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } From d755e394e97e5eef8ac46e191a1c3b01f0a81e48 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 12:34:33 +0100 Subject: [PATCH 538/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index c29a278..449a162 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -186,7 +186,9 @@ def jiraticketexists(String jiraticket) { def extractjiraticketinfo(String line) { def List jiratickets = [] - + boolean exists = false + String URI = '' + while (true) { line = line.trim() def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 From 2cf59337afb680a4551bc7fe787dff099d03bcd7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 13:07:57 +0100 Subject: [PATCH 539/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 449a162..43e42cb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -197,7 +197,8 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - jiratickets.add([jiraticket, URI]) + def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] + jiratickets.add(jiraticketmap) line = match1.group(2) } else { break @@ -209,7 +210,8 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - jiratickets.add([jiraticket, URI]) + def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] + jiratickets.add(jiraticketmap) line = match2.group(3) } else { break @@ -221,7 +223,8 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - jiratickets.add([jiraticket, URI]) + def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] + jiratickets.add(jiraticketmap) line = match3.group(3) } else { break From 86d40f76887a5af654c7ac91e789fdf3222f5cc0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 14:46:50 +0100 Subject: [PATCH 540/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 43e42cb..7d66151 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -361,7 +361,7 @@ def mergereport(String reportsubfolder, merges, boolean iscomponent = true) { def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { dir(reportsubfolder) { String revertlist = "Revert list:\n" - revertssorted = reverts.sort() + revertssorted = reverts.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (arevert in revertssorted) { revertlist += "${arevert.key}:\n" for (revertlistcomment in arevert.value) { @@ -375,7 +375,7 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" - ticketedsorted = ticketed.sort() + ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (aticketed in ticketedsorted) { ticketedlist += "${aticketed.key}:\n" for (ticketedlistcomment in aticketed.value) { @@ -400,7 +400,7 @@ def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = t def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { dir(reportsubfolder) { String ticketlist = "Ticket list:\n" - ticketssorted = tickets.sort() + ticketssorted = tickets.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (ticketnumber in ticketssorted) { ticketlist += "${ticketnumber.key} (${ticketnumber.value})\n" } From 19cfa866a4a422a67676bb21c187e08bd882e97d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:05:27 +0100 Subject: [PATCH 541/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 7d66151..d959b0f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -444,7 +444,8 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { comments[dr.comment] += 1 } } - reverts[ticket] = comments + reverts[ticket] = comments + echo "ticket: ${ticket} revert[ticket]: ${reverts[ticket]}" if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 } else { From 783f5e2b42c6df2113565a445f3da1a1430495b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:15:18 +0100 Subject: [PATCH 542/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index d959b0f..307fd48 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -363,7 +363,7 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { String revertlist = "Revert list:\n" revertssorted = reverts.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (arevert in revertssorted) { - revertlist += "${arevert.key}:\n" + revertlist += "${arevert.key['jiraticket']}: ${arevert.key['uri']}\n" for (revertlistcomment in arevert.value) { revertlist += "${revertlistcomment.key} (${revertlistcomment.value})\n" } @@ -377,7 +377,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (aticketed in ticketedsorted) { - ticketedlist += "${aticketed.key}:\n" + ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" for (ticketedlistcomment in aticketed.value) { ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" } @@ -386,6 +386,8 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) } } +ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] + def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { dir(reportsubfolder) { String unticketedlist = "Unticketed list:\n" @@ -402,7 +404,7 @@ def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { String ticketlist = "Ticket list:\n" ticketssorted = tickets.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber.key} (${ticketnumber.value})\n" + ticketlist += "${ticketnumber.key['jiraticket']}: ${ticketnumber.key['uri']} (${ticketnumber.value})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } From 1ff0e8298b30e5b42b5c3869c6af0d80e3ab52fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:17:10 +0100 Subject: [PATCH 543/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 307fd48..bb86e36 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -386,7 +386,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) } } -ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] +//ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { dir(reportsubfolder) { From 05271844bbd4f43c2863faebe16d390aee8c4f77 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:23:30 +0100 Subject: [PATCH 544/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index bb86e36..e35cd8d 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,6 +377,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } for (aticketed in ticketedsorted) { + echo "aticketed: ${aticketed}" ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" for (ticketedlistcomment in aticketed.value) { ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" @@ -447,7 +448,6 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } } reverts[ticket] = comments - echo "ticket: ${ticket} revert[ticket]: ${reverts[ticket]}" if (!tickets.containsKey(ticket)) { tickets[ticket] = 1 } else { From d154a643e2bfe378b9472e865b8cd657c1f8e56a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:31:40 +0100 Subject: [PATCH 545/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e35cd8d..a7ca611 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -376,6 +376,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + echo "ticketedsorted: ${ticketedsorted}" for (aticketed in ticketedsorted) { echo "aticketed: ${aticketed}" ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" From 79a9eecc51aa05306b088ae1c8d21b2eb94ba033 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 15:37:27 +0100 Subject: [PATCH 546/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a7ca611..adefdfc 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -375,7 +375,8 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" - ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + echo "ticketed: ${ticketed}" + ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } echo "ticketedsorted: ${ticketedsorted}" for (aticketed in ticketedsorted) { echo "aticketed: ${aticketed}" From 15ae9f815f3680b0a5103929eace3aca17eec691 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 16:25:52 +0100 Subject: [PATCH 547/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index adefdfc..1b297de 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -375,8 +375,14 @@ def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { dir(reportsubfolder) { String ticketedlist = "Ticketed list:\n" - echo "ticketed: ${ticketed}" - ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + echo "ticketed: ${ticketed}" + for (ticket in ticketed) { + echo "|${ticket.key['jiraticket']|\n|${ticket.key['uri']|\n" + for (comment in ticket.value) { + echo "|${comment.key}|${comment.value}|\n" + } + } + ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } echo "ticketedsorted: ${ticketedsorted}" for (aticketed in ticketedsorted) { echo "aticketed: ${aticketed}" From 39b849c61caf2315b059c04fd3f6557fe9359975 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 16:38:32 +0100 Subject: [PATCH 548/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1b297de..5d290b9 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,7 +377,7 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" echo "ticketed: ${ticketed}" for (ticket in ticketed) { - echo "|${ticket.key['jiraticket']|\n|${ticket.key['uri']|\n" + echo "|${ticket.key['jiraticket']}|\n|${ticket.key['uri']}|\n" for (comment in ticket.value) { echo "|${comment.key}|${comment.value}|\n" } From 9062919a7f4b1a43d6cd87d65218574eaeb1c3f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 16:56:19 +0100 Subject: [PATCH 549/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 5d290b9..06ade92 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -377,9 +377,9 @@ def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) String ticketedlist = "Ticketed list:\n" echo "ticketed: ${ticketed}" for (ticket in ticketed) { - echo "|${ticket.key['jiraticket']}|\n|${ticket.key['uri']}|\n" + echo "|${ticket.key['jiraticket']}(${ticket.key['jiraticket'].getClass()})|\n|${ticket.key['uri']}(${ticket.key['uri'].getClass()})|\n" for (comment in ticket.value) { - echo "|${comment.key}|${comment.value}|\n" + echo "|${comment.key}(${comment.key.getClass()})|${comment.value}(${comment.value.getClass()})|\n" } } ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } From 1b247ccb774aed23a3dde6a33050c4c800058459 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:38:16 +0100 Subject: [PATCH 550/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 262 +++++++++++++++++++++-------- 1 file changed, 195 insertions(+), 67 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 06ade92..9adc784 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -184,6 +184,16 @@ def jiraticketexists(String jiraticket) { return [true, URI] } +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } +} + def extractjiraticketinfo(String line) { def List jiratickets = [] boolean exists = false @@ -197,8 +207,10 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] - jiratickets.add(jiraticketmap) + def JiraTicketInfo j = new JiraTicketInfo + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) line = match1.group(2) } else { break @@ -210,8 +222,10 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] - jiratickets.add(jiraticketmap) + def JiraTicketInfo j = new JiraTicketInfo + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) line = match2.group(3) } else { break @@ -223,8 +237,10 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def jiraticketmap = ['jiraticket' : jiraticket, 'uri' : URI] - jiratickets.add(jiraticketmap) + def JiraTicketInfo j = new JiraTicketInfo + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) line = match3.group(3) } else { break @@ -347,73 +363,137 @@ def addedcomponent(latermetaitems, key) { summaryappend("Added ") } -def mergereport(String reportsubfolder, merges, boolean iscomponent = true) { +class MergeReportRecord { + String comment + int duplicates + + MergeReportRecord() { + comment = '' + duplicates = 0 + } +} + +def mergereport(String reportsubfolder, LinkedHashMap merges, boolean iscomponent = true) { dir(reportsubfolder) { - String mergelist = "Merge list:\n" - mergessorted = merges.sort() + def String mergelist = "Merge list:\n" + def mergessorted = merges.sort() for (amerge in mergessorted) { - mergelist += "${amerge.key}\n" + def MergeReportRecord m = amerge.value + mergelist += "${m.comment} (${m.duplicates()})\n" } writeFile(file: "mergelist.txt", text: mergelist) } } -def revertsreport(String reportsubfolder, reverts, boolean iscomponent = true) { +class CommentsRecord { + String comment + int duplicates + + CommentsRecord() { + comment = '' + duplicates = 0 + } +} + +class RevertsReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + RevertsReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def revertsreport(String reportsubfolder, LinkedHashMap reverts, boolean iscomponent = true) { dir(reportsubfolder) { - String revertlist = "Revert list:\n" - revertssorted = reverts.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + def String revertlist = "Revert list:\n" + def revertssorted = reverts.sort() for (arevert in revertssorted) { - revertlist += "${arevert.key['jiraticket']}: ${arevert.key['uri']}\n" - for (revertlistcomment in arevert.value) { - revertlist += "${revertlistcomment.key} (${revertlistcomment.value})\n" + def RevertsReportRecord r = arevert.value + revertlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + revertlist += "${commentrecord.comment} (${commentrecord.duplicates})\n" } } writeFile(file: "revertlist.txt", text: revertlist) } } -def ticketedreport(String reportsubfolder, ticketed, boolean iscomponent = true) { +class TicketedReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + TicketedReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def ticketedreport(String reportsubfolder, LinkedHashMap ticketed, boolean iscomponent = true) { dir(reportsubfolder) { - String ticketedlist = "Ticketed list:\n" - echo "ticketed: ${ticketed}" - for (ticket in ticketed) { - echo "|${ticket.key['jiraticket']}(${ticket.key['jiraticket'].getClass()})|\n|${ticket.key['uri']}(${ticket.key['uri'].getClass()})|\n" - for (comment in ticket.value) { - echo "|${comment.key}(${comment.key.getClass()})|${comment.value}(${comment.value.getClass()})|\n" - } - } - ticketedsorted = ticketed.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } - echo "ticketedsorted: ${ticketedsorted}" + def String ticketedlist = "Ticketed list:\n" + def ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { - echo "aticketed: ${aticketed}" - ticketedlist += "${aticketed.key['jiraticket']}: ${aticketed.key['uri']}\n" - for (ticketedlistcomment in aticketed.value) { - ticketedlist += "${ticketedlistcomment.key} (${ticketedlistcomment.value})\n" + def TicketedReportRecord r = aticket.value + ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + ticketedlist += "${commentrecord.comment} (${commentrecord.duplicates})\n" } } writeFile(file: "ticketedlist.txt", text: ticketedlist) } } -//ticket: [jiraticket:GIS-7312, uri:https://jira.gisapps.co.uk/browse/GIS-7312] revert[ticket]: [- The download of the latest 'sentinel super pro' drivers and placed next to the original dongle drivers.:1] +class UnticketedReportRecord { + String comment + int duplicates + + UnticketedReportRecord() { + comment = '' + duplicates = 0 + } +} -def unticketedreport(String reportsubfolder, unticketed, boolean iscomponent = true) { +def unticketedreport(String reportsubfolder, LinkedHashMap unticketed, boolean iscomponent = true) { dir(reportsubfolder) { - String unticketedlist = "Unticketed list:\n" - unticketedsorted = unticketed.sort() + def String unticketedlist = "Unticketed list:\n" + def unticketedsorted = unticketed.sort() for (anunticketed in unticketedsorted) { - unticketedlist += "${anunticketed.key} (${anunticketed.value})\n" + UnticketedReportRecord u = anunticketed.value + unticketedlist += "${u.comment} (${u.duplicates})\n" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } } +class TicketsReportRecord { + String jiraticket + String ticketURI + int duplicates + + TicketsReportRecord() { + jiraticket = '' + ticketURI = '' + duplicates = 0 + } +} + def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { dir(reportsubfolder) { String ticketlist = "Ticket list:\n" - ticketssorted = tickets.toSorted { a, b -> a.key['jiraticket'] <=> b.key['jiraticket'] } + ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { - ticketlist += "${ticketnumber.key['jiraticket']}: ${ticketnumber.key['uri']} (${ticketnumber.value})\n" + TicketsReportRecord t = ticketnumber.value + ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } @@ -438,60 +518,108 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { for (dr in differencerecords) { if (dr.type == "Merge") { - merges[dr.mergeinfo] = dr + def MergeReportRecord m + if (!merges.containsKey(dr.mergeinfo)) { + m = new MergeReportRecord() + m.comment = dr.mergeinfo + m.duplicate = 1 + } else { + m = merges[dr.mergeinfo] + m.duplicate += 1 + } + merges[dr.mergeinfo] = m } if (dr.type == "Revert") { + def RevertsReportRecord r for (ticket in dr.jiraticketinfo) { - def comments = [:] - if (!reverts.containsKey(ticket)) { - reverts[ticket] = [:] - comments = reverts[ticket] - comments[dr.comment] = 1 + JiraTicketInfo j = ticket + if (!reverts.containsKey(j.jiranumber)) { + r = new RevertsReportRecord() + r.jiraticket = j.jiranumber + r.ticketURI = j.uri + CommentsRecord c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c } else { - comments = reverts[ticket] - if (!comments.containsKey(dr.comment)) { - comments[dr.comment] = 1 + r = reverts[j.jiranumber] + def CommentsRecord c + if (!r.commentscollection.containsKey(dr.comment)) { + c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c } else { - comments[dr.comment] += 1 + c = r.commentscollection[dr.comment] + c.duplicates += 1 } + r.commentscollection[dr.comment] = c } - reverts[ticket] = comments - if (!tickets.containsKey(ticket)) { - tickets[ticket] = 1 + reverts[j.jiranumber] = r + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 } else { - tickets[ticket] += 1 + t = tickets[j.jiranumber] + t.duplicates += 1 } + tickets[j.jiranumber] = t } } if (dr.type == "Ticketed") { + def TicketedReportRecord t for (ticket in dr.jiraticketinfo) { - def comments = [:] - if (!ticketed.containsKey(ticket)) { - ticketed[ticket] = [:] - comments = ticketed[ticket] - comments[dr.comment] = 1 + JiraTicketInfo j = ticket + if (!ticketed.containsKey(j.jiranumber)) { + t = new TicketedReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + CommentsRecord c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c } else { - comments = ticketed[ticket] - if (!comments.containsKey(dr.comment)) { - comments[dr.comment] = 1 + t = ticketed[j.jiranumber] + def CommentsRecord c + if (!t.commentscollection.containsKey(dr.comment)) { + c = mew CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c } else { - comments[dr.comment] += 1 + c = t.commentscollection[dr.comment] + c.duplicates += 1 } + t.commentscollection[dr.comment] = c } - ticketed[ticket] = comments - if (!tickets.containsKey(ticket)) { - tickets[ticket] = 1 + ticketed[j.jiranumber] = t + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 } else { - tickets[ticket] += 1 + t = tickets[j.jiranumber] + t.duplicates += 1 } + tickets[j.jiranumber] = t } } if (dr.type == "Unticketed") { + UnticketedReportRecord u if (!unticketed.containsKey(dr.comment)) { - unticketed[dr.comment] = 1 + u = new UnticketedReportRecord() + u.comment = dr.comment + u.duplicates = 1 } else { - unticketed[dr.comment] += 1 + u = unticketed[dr.comment] + u.duplicates += 1 } + unticketed[dr.comment] = u } } String reportsubfolder = "" From c7536f377b217c189eb09499d12838bc1cf44312 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:43:43 +0100 Subject: [PATCH 551/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9adc784..a542edb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -207,7 +207,7 @@ def extractjiraticketinfo(String line) { jiraticket = match1.group(1) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def JiraTicketInfo j = new JiraTicketInfo + def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) @@ -222,7 +222,7 @@ def extractjiraticketinfo(String line) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def JiraTicketInfo j = new JiraTicketInfo + def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) @@ -237,7 +237,7 @@ def extractjiraticketinfo(String line) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI) = jiraticketexists(jiraticket) if (exists) { - def JiraTicketInfo j = new JiraTicketInfo + def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) From b3188c775247a4a2de587a47b00571ac222cd716 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:47:56 +0100 Subject: [PATCH 552/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a542edb..3a2538e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -596,17 +596,17 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { t.commentscollection[dr.comment] = c } ticketed[j.jiranumber] = t - def TicketsReportRecord t + def TicketsReportRecord ti if (!tickets.containsKey(j.jiranumber)) { - t = new TicketsReportRecord() - t.jiraticket = j.jiranumber - t.ticketURI = j.uri - t.duplicates = 1 + ti = new TicketsReportRecord() + ti.jiraticket = j.jiranumber + ti.ticketURI = j.uri + ti.duplicates = 1 } else { - t = tickets[j.jiranumber] - t.duplicates += 1 + ti = tickets[j.jiranumber] + ti.duplicates += 1 } - tickets[j.jiranumber] = t + tickets[j.jiranumber] = ti } } if (dr.type == "Unticketed") { From 28dc2231bf161b289c5b678603bba78e6b91c1a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 20:56:07 +0100 Subject: [PATCH 553/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3a2538e..a1ed656 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -192,6 +192,10 @@ class JiraTicketInfo { jiranumber = '' uri = '' } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } } def extractjiraticketinfo(String line) { @@ -522,10 +526,10 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { if (!merges.containsKey(dr.mergeinfo)) { m = new MergeReportRecord() m.comment = dr.mergeinfo - m.duplicate = 1 + m.duplicates = 1 } else { m = merges[dr.mergeinfo] - m.duplicate += 1 + m.duplicates += 1 } merges[dr.mergeinfo] = m } From 7ff591f1fa1db05fa59294bdc0a8aba2ed382560 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:04:00 +0100 Subject: [PATCH 554/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 40 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index a1ed656..4401059 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -10,6 +10,20 @@ tempfolder = "${WORKSPACE}/temp" reportfolder = "${WORKSPACE}/reports" +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } +} + class DifferenceRecord { String productname String subcomponentname @@ -19,7 +33,7 @@ class DifferenceRecord { String sha String type String mergeinfo - List jiraticketinfo + List jiraticketinfo String comment DifferenceRecord() { @@ -31,7 +45,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" - jiraticketinfo = [] + jiraticketinfo = new List comment = "" } @@ -184,20 +198,6 @@ def jiraticketexists(String jiraticket) { return [true, URI] } -class JiraTicketInfo { - String jiranumber - String uri - - JiraTicketInfo() { - jiranumber = '' - uri = '' - } - - def String dump() { - return "jiranumber: ${jiranumber} uri: ${uri}" - } -} - def extractjiraticketinfo(String line) { def List jiratickets = [] boolean exists = false @@ -541,7 +541,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { r = new RevertsReportRecord() r.jiraticket = j.jiranumber r.ticketURI = j.uri - CommentsRecord c = mew CommentsRecord() + CommentsRecord c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 r.commentscollection[dr.comment] = c @@ -549,7 +549,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { r = reverts[j.jiranumber] def CommentsRecord c if (!r.commentscollection.containsKey(dr.comment)) { - c = mew CommentsRecord() + c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 r.commentscollection[dr.comment] = c @@ -581,7 +581,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { t = new TicketedReportRecord() t.jiraticket = j.jiranumber t.ticketURI = j.uri - CommentsRecord c = mew CommentsRecord() + CommentsRecord c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 t.commentscollection[dr.comment] = c @@ -589,7 +589,7 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { t = ticketed[j.jiranumber] def CommentsRecord c if (!t.commentscollection.containsKey(dr.comment)) { - c = mew CommentsRecord() + c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 t.commentscollection[dr.comment] = c From 348620371002aa07e3f964979188d75bfb2a08cf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:15:41 +0100 Subject: [PATCH 555/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 4401059..3916d61 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -45,7 +45,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" - jiraticketinfo = new List + jiraticketinfo = new List() comment = "" } From 4b2b4018f787e05f177d68d559e2acf7e74ffda7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:29:06 +0100 Subject: [PATCH 556/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3916d61..1fb6444 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -45,7 +45,7 @@ class DifferenceRecord { sha = "" type = "" mergeinfo = "" - jiraticketinfo = new List() + jiraticketinfo = [] comment = "" } From 5c1c461bcd07812720d7c5a25dfd904cf60378a3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:37:55 +0100 Subject: [PATCH 557/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1fb6444..08017cb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -50,7 +50,11 @@ class DifferenceRecord { } def String dump() { - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketinfo}\ncomment: ${comment}" + String jiraticketstring = '' + for (j in jiraticketinfo) { + jiraticketstring += j.dump() + " " + } + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -383,7 +387,7 @@ def mergereport(String reportsubfolder, LinkedHashMap def mergessorted = merges.sort() for (amerge in mergessorted) { def MergeReportRecord m = amerge.value - mergelist += "${m.comment} (${m.duplicates()})\n" + mergelist += "${m.comment} (${m.duplicates})\n" } writeFile(file: "mergelist.txt", text: mergelist) } From bac02149cecb8723e138ce062f6e51035dd2524d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 21:48:53 +0100 Subject: [PATCH 558/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 08017cb..20c2af6 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -425,7 +425,8 @@ def revertsreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 21:50:21 +0100 Subject: [PATCH 559/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 20c2af6..63ba65e 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -425,7 +425,7 @@ def revertsreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:03:12 +0100 Subject: [PATCH 560/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 63ba65e..8bd1a74 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -209,6 +209,11 @@ def extractjiraticketinfo(String line) { while (true) { line = line.trim() + def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line + def commamatch = (line =~ commapattern) + if (commamatch.find()) { + line = commamatch.group(1).trim() + } def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) if (match1.find()) { @@ -450,7 +455,7 @@ def ticketedreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:11:00 +0100 Subject: [PATCH 561/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 8bd1a74..39b57e6 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -460,7 +460,8 @@ def ticketedreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:42:21 +0100 Subject: [PATCH 562/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 39b57e6..60a4c66 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -464,6 +464,7 @@ def ticketedreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 22:47:09 +0100 Subject: [PATCH 563/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 60a4c66..3c4aefb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -337,7 +337,7 @@ def List processtext(meta, String text) { text.split('\n').each { DifferenceRecord dr = fillrecord(meta, it.trim()) differencerecords.add(dr) - echo "${dr.dump()}" + //echo "${dr.dump()}" } return differencerecords } From 39e4ff5409bd48803dd52e493f7685505c8f630e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 22:50:03 +0100 Subject: [PATCH 564/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3c4aefb..9443fd1 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -433,6 +433,7 @@ def revertsreport(String reportsubfolder, LinkedHashMap Date: Sat, 6 Jul 2019 23:01:36 +0100 Subject: [PATCH 565/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9443fd1..b4f438f 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -189,7 +189,8 @@ def extractmerges(String line) { def jiraticketexists(String jiraticket) { def String URI = "${JIRA_URI}/browse/${jiraticket}" - def get = new URL(URI).openConnection(); + def String URIforsummary = "${URI}?fields=summary" + def get = new URL(URIforsummary).openConnection(); get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) From b2a891e9cc36fff7b28f5ebec31269f13f85cbc8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 23:17:12 +0100 Subject: [PATCH 566/589] Add more difference infrastructure; Check with Jira --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index b4f438f..6d5c2fb 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -521,7 +521,7 @@ def differencerecordsreport(String reportsubfolder, differencerecords, boolean i dir(reportsubfolder) { String recordlist = "" for (dr in differencerecords) { - recordlist += dr.dump() + "\n" + recordlist += dr.dump() + "\n\n" } writeFile(file: "recordlist.txt", text: recordlist) } From 4a796b33abb626ada8b8e45e47058a53eeada34f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 6 Jul 2019 23:57:39 +0100 Subject: [PATCH 567/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6d5c2fb..0b056cf 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -101,6 +101,7 @@ def showmetaitems(metaitems) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" if (sourcerepo) { // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component if (fileExists(checkoutfolder)) { From e40949419a5d18721bc66f27b73b0342242d773d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 00:06:45 +0100 Subject: [PATCH 568/589] Add more difference infrastructure --- differences/dometahierarchy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 0b056cf..3c97076 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -127,6 +127,7 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + echo "Cloning/updating source repo in folder: ${uniquefolderforsourcerepo}" checkoutsourcerepo(meta, uniquefolderforsourcerepo) return uniquefolderforsourcerepo From 85d7f751da8e638bd2d77331810978f3f29d4a0f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 00:27:10 +0100 Subject: [PATCH 569/589] Handle components that have no source repo --- differences/dometahierarchy.groovy | 57 ++++++++++++++++++------------ 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3c97076..6336c33 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -99,27 +99,32 @@ def showmetaitems(metaitems) { echo info } -def checkoutsourcerepo(buildmeta, String checkoutfolder) { +def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { + boolean checkedout = false def String sourcerepo = buildmeta.gitinfo.remoteurl - echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" - if (sourcerepo) { - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - """ - ) + if (sourcerepo?.trim()) { + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + """ + ) + } } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } } } + checkedout = true } + return checkedout } def String getcomponent(meta) { @@ -127,10 +132,9 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - echo "Cloning/updating source repo in folder: ${uniquefolderforsourcerepo}" - checkoutsourcerepo(meta, uniquefolderforsourcerepo) + bool checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) - return uniquefolderforsourcerepo + return [checkedout, uniquefolderforsourcerepo] } def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { @@ -665,12 +669,19 @@ def compare(earlymetaitems, latermetaitems, key) { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") - def uniquefolderforsourcerepo = getcomponent(earlymeta) + def boolean repoavailable + def String uniquefolderforsourcerepo + (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) - (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + List differencerecords = [] + if (repoavailable) { + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - summaryappend(" Items: ${differencerecords.size()} ") - report(earlymeta, differencerecords) + summaryappend(" Items: ${differencerecords.size()} ") + report(earlymeta, differencerecords) + } else { + summaryappend(" No source repo ") + } summaryappend("\n") return [result, differencerecords]; From a0fcc01a0d9d808e8453b523458ab89bc48f9d32 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 00:30:39 +0100 Subject: [PATCH 570/589] Handle components that have no source repo --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6336c33..b7d4615 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -132,7 +132,7 @@ def String getcomponent(meta) { def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - bool checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) + boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) return [checkedout, uniquefolderforsourcerepo] } From 06591a0daaddcc2da9570ce230c86216c355c1ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 07:44:38 +0100 Subject: [PATCH 571/589] Handle the no change case - the early and later sha codes match --- differences/dometahierarchy.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index b7d4615..e515c06 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -342,9 +342,12 @@ def List processtext(meta, String text) { def List differencerecords = [] text.split('\n').each { - DifferenceRecord dr = fillrecord(meta, it.trim()) - differencerecords.add(dr) - //echo "${dr.dump()}" + def line = it + if (line?.trim()) { + def DifferenceRecord dr = fillrecord(meta, line.trim()) + differencerecords.add(dr) + //echo "${dr.dump()}" + } } return differencerecords } @@ -363,6 +366,9 @@ def getdifferencerecords(earlymeta, latermeta, String sourcepath) { summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) differencerecords = processtext(earlymeta, differencetext) + if (earlyrevision == laterrevision) { + summaryappend(" No change") + } } else { summaryappend("The early and later git urls don't match") echo "The early and later git urls don't match" From aecd252a57158eb20babe62cc40bb43be5e433b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 09:04:07 +0100 Subject: [PATCH 572/589] Normalise Jira ticket number --- differences/dometahierarchy.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e515c06..dccb5b4 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -225,6 +225,7 @@ def extractjiraticketinfo(String line) { def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) + jiraticket = jiraticket.toUpperCase() (exists, URI) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() @@ -240,6 +241,7 @@ def extractjiraticketinfo(String line) { def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) + "-" + match2.group(2) + jiraticket = jiraticket.toUpperCase() (exists, URI) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() @@ -255,6 +257,7 @@ def extractjiraticketinfo(String line) { def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) + "-" + match3.group(2) + jiraticket = jiraticket.toUpperCase() (exists, URI) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() From 877f022678fc67bc971cb69a18b88b8c2b508a79 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 10:44:26 +0100 Subject: [PATCH 573/589] Handle Jira ticket redirects - as a result of a jira ticket being moved to another project --- differences/dometahierarchy.groovy | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index dccb5b4..9258da3 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,3 +1,5 @@ +import groovy.json.JsonSlurper + markunstable = false unstablereason = "" @@ -203,10 +205,15 @@ def jiraticketexists(String jiraticket) { get.setRequestProperty( 'Accept', 'application/json' ) def getRC = get.getResponseCode(); + def String redirectedjiraticket = jiraticket if(!getRC.equals(200)) { - return [false, ''] + return [false, '', jiraticket] + } else { + def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) + echo "Response: ${jsonresponse}" + redirectedjiraticket = jsonresponse['key'] } - return [true, URI] + return [true, URI, redirectedjiraticket] } def extractjiraticketinfo(String line) { @@ -225,8 +232,7 @@ def extractjiraticketinfo(String line) { def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) - jiraticket = jiraticket.toUpperCase() - (exists, URI) = jiraticketexists(jiraticket) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket @@ -241,8 +247,7 @@ def extractjiraticketinfo(String line) { def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) + "-" + match2.group(2) - jiraticket = jiraticket.toUpperCase() - (exists, URI) = jiraticketexists(jiraticket) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket @@ -257,8 +262,7 @@ def extractjiraticketinfo(String line) { def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) + "-" + match3.group(2) - jiraticket = jiraticket.toUpperCase() - (exists, URI) = jiraticketexists(jiraticket) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket From 8ce4442dd68c7e422a1f4c2abcb1657062293975 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 7 Jul 2019 12:15:53 +0100 Subject: [PATCH 574/589] Handle Jira ticket redirects - as a result of a jira ticket being moved to another project --- differences/dometahierarchy.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 9258da3..11b3733 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -197,8 +197,8 @@ def extractmerges(String line) { def jiraticketexists(String jiraticket) { def String URI = "${JIRA_URI}/browse/${jiraticket}" - def String URIforsummary = "${URI}?fields=summary" - def get = new URL(URIforsummary).openConnection(); + def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" + def get = new URL(RestfulURI).openConnection(); get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) @@ -210,7 +210,6 @@ def jiraticketexists(String jiraticket) { return [false, '', jiraticket] } else { def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) - echo "Response: ${jsonresponse}" redirectedjiraticket = jsonresponse['key'] } return [true, URI, redirectedjiraticket] From 8af57f80b93098579f7c2e4556c0c6eacd0da00c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 18 Jul 2019 11:22:06 +0100 Subject: [PATCH 575/589] Provide logging to debug why import of Print Server artifacts results in duff associated meta data --- meta/folder.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index f8c033b..b559458 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -571,6 +571,7 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJson(String filepath) { + script.echo("Deserialising JSON at file: ${filepath}" def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } @@ -617,6 +618,8 @@ class MetaInfo extends GISJsonSerialised { def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") } RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) From dbe2cdd5fdbef3e21bf5d7c0a97f69503576ea11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 18 Jul 2019 11:34:23 +0100 Subject: [PATCH 576/589] Provide logging to debug why import of Print Server artifacts results in duff associated meta data --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index b559458..ddc99c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -571,7 +571,7 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJson(String filepath) { - script.echo("Deserialising JSON at file: ${filepath}" + script.echo("Deserialising JSON at file: ${filepath}") def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } From aaba6bf78ef17b1c84d33490611bb17f4d9ba26a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:43:21 +0100 Subject: [PATCH 577/589] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 738c04a..1898594 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -92,9 +92,10 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { } } -def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { +def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") + bool exceptionduringcopyartifacts = false for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -102,22 +103,30 @@ def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, Strin } // Copy relevant artifacts to temp folder echo "Copying artifacts: ${filterpath} to ${tempfolder}" - copyArtifacts( - filter: filterpath, - projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - ], - target: tempfolder - ) - // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.offsetpath - dir(originalrootdir) { - sh("mv * ${repofolder}") + try { + copyArtifacts( + filter: filterpath, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + ], + target: tempfolder + ) + } catch (Exception e) { + echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" + exceptionduringcopyartifacts = true + summaryappend("Does not exist: ${artifactfilter}," + } else { + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } } summaryappend("${artifactfilter},") } + return exceptionduringcopyartifacts } def releasegitadd() { @@ -357,9 +366,9 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp) - - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + } summaryappend("\n") } From 31012ff5189f729a1f6c9b349ff5fc4aa0f61c13 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:51:29 +0100 Subject: [PATCH 578/589] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1898594..10fc165 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -117,7 +117,8 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true summaryappend("Does not exist: ${artifactfilter}," - } else { + } + if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { From 6539223840f6c7c0a0ec7bb2f3f8977760515a3a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:53:31 +0100 Subject: [PATCH 579/589] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 10fc165..c1c4277 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -116,7 +116,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true - summaryappend("Does not exist: ${artifactfilter}," + summaryappend("Does not exist: ${artifactfilter},") } if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position From 93b2b46cd0c2a03fb0dbbd7311fdaa56c0facfdd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 14:09:53 +0100 Subject: [PATCH 580/589] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index c1c4277..0e7809e 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -95,7 +95,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - bool exceptionduringcopyartifacts = false + boolean exceptionduringcopyartifacts = false for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -116,7 +116,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true - summaryappend("Does not exist: ${artifactfilter},") + summaryappend("Does not exist:") } if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position From e9ce58fc62c3201c0186d528c66e077ce193c4c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 14:33:01 +0100 Subject: [PATCH 581/589] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0e7809e..1b21bcf 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -95,7 +95,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - boolean exceptionduringcopyartifacts = false + def boolean result = true for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -115,10 +115,10 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde ) } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" - exceptionduringcopyartifacts = true + result = false summaryappend("Does not exist:") } - if (!exceptionduringcopyartifacts) { + if (result) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { @@ -127,7 +127,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } summaryappend("${artifactfilter},") } - return exceptionduringcopyartifacts + return result } def releasegitadd() { From b6528264be819a05a38b482bbc10bc68d7c22a48 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 25 Jul 2019 17:21:18 +0100 Subject: [PATCH 582/589] GIS-1621 - Import print server source such that it is registered in the meta data --- configuration/configuration.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 1864ba9..b8c0b47 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -8,8 +8,10 @@ def registerPipelineScriptConfiguration(meta) { registerConfigurationInfo(meta, 'pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } -def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, String filter = '') { + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } From 6b7eb941a4597f843e56529fe13122f4052bbea6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 30 Jul 2019 22:48:18 +0100 Subject: [PATCH 583/589] GIS-1621 - Allow installer builder handle a list of artifact keys or a single string --- install/installhelper.groovy | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index eebbd34..0d1863e 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -28,11 +28,16 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { - def artifactKey = installerDetails.get(installer)[1] - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) - if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) - artifactFetched.put(artifactKey, importmeta) + def artifactKeys = installerDetails.get(installer)[1] + if (obj instanceof java.lang.Integer) { + artifactKeys = [ artifactKeys ] + } + for (artifactKey in artifactKeys) { + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + if (artifactFetched.get(artifactKey) == null) { + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) + artifactFetched.put(artifactKey, importmeta) + } } } return artifactFetched From 09e3bfcd0cf62bc58660694335997fee8a9fc6a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Jul 2019 08:03:54 +0100 Subject: [PATCH 584/589] GIS-1621 - Allow installer builder handle a list of artifact keys or a single string --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 0d1863e..75ba5d6 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -29,7 +29,7 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] - if (obj instanceof java.lang.Integer) { + if (artifactKeys instanceof java.lang.String) { artifactKeys = [ artifactKeys ] } for (artifactKey in artifactKeys) { From e7b86d95226095c5f2f601b6e08a6710705aa942 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 1 Aug 2019 08:55:18 +0100 Subject: [PATCH 585/589] Perform a shallow clone check on repos being used to generate version numbers --- versioncontrol/git.groovy | 15 +++++++++++++++ versionnumbering/generate.groovy | 9 ++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 3ea97ec..0e27faf 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -4,6 +4,21 @@ import hudson.plugins.git.extensions.impl.* clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") +def boolean isShallowClonedRepo(String gitrootpath) { + def boolean isshallowclone = true + + dir( gitrootpath) { + if (isUnix()) { + isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() + } + else { + isshallowclone = bat(returnStdout: true, script: "@git rev-parse --is-shallow-repository").trim().toBoolean() + } + echo "Repo at ${gitrootpath} is shallow cloned: Change count: ${isshallowclone}" + } + return isshallowclone +} + String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { return clh.getCommandOutput('git rev-parse HEAD') diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f4f1bdc..2e4f60d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,4 +1,5 @@ - +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + def int maxversionnumber() { def int MAXVERSIONNUMBER = 65535 return MAXVERSIONNUMBER @@ -15,6 +16,12 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String gitrevision def String remoteinfo def String remoteurl + + def boolean isShallowCloned = gh.isShallowClonedRepo(gitrootpath) + if (isShallowCloned) { + error("Cannot generate a version number from a shallow cloned repo at ${gitrootpath}") + } + dir( gitrootpath) { def int changecount = 0 if (isUnix()) { From b6aa653aeb590a37e2766b89cfc0e08106dee7eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 1 Aug 2019 08:57:53 +0100 Subject: [PATCH 586/589] Perform a shallow clone check on repos being used to generate version numbers --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 0e27faf..bb7c0ce 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -14,7 +14,7 @@ def boolean isShallowClonedRepo(String gitrootpath) { else { isshallowclone = bat(returnStdout: true, script: "@git rev-parse --is-shallow-repository").trim().toBoolean() } - echo "Repo at ${gitrootpath} is shallow cloned: Change count: ${isshallowclone}" + echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" } return isshallowclone } From daad8d0118f20c1cef4cf98c52cb3eec0ce8b400 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 9 Aug 2019 13:04:29 +0100 Subject: [PATCH 587/589] GIS-7717 GIS-7718 - Select the host dependent certificate --- install/installhelper.groovy | 12 ++++++++++++ textutils/subst.groovy | 8 ++++++++ 2 files changed, 20 insertions(+) create mode 100644 textutils/subst.groovy diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 75ba5d6..6fcdc6c 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -97,4 +97,16 @@ def checkISMfile(String ISMfile) { } } +certificatethumbprints = [:] +certificatethumbprints['floreana'] = '0FDEB0A227982C877146CA595091173AC9655479' // Authoring takes place on this machine - so this is what should be in the ISM file already +certificatethumbprints['hawaii'] = '49FD1B361CFF35014B81940C5B046543045DD34B' + +def selectSigningCertificate(String nodeName, String ISMfile) { + def masterpattern = '0FDEB0A227982C877146CA595091173AC9655479' + echo "Substituting occurrences of ${masterpattern} with ${certificatethumbprints[nodeName]} in ISM file: ${ISMfile}" + + textutils_subst = load("${autobuilder.commonscripts()}/textutils/subst.groovy") + textutils_subst.substitute(ISMfile, masterpattern, certificatethumbprints[nodeName]) +} + return this; diff --git a/textutils/subst.groovy b/textutils/subst.groovy new file mode 100644 index 0000000..aae44c6 --- /dev/null +++ b/textutils/subst.groovy @@ -0,0 +1,8 @@ + +def substitute(String filepath, String pattern, String subst) { + fileText = readFile(filepath) + def converted = fileText.replaceAll(pattern, subst) + writeFile(file: filepath, text: converted +} + +return this; \ No newline at end of file From 3cbc5bbf9337fd9ed25e928a8e298712a149bb1a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 9 Aug 2019 13:14:02 +0100 Subject: [PATCH 588/589] GIS-7717 GIS-7718 - Select the host dependent certificate --- textutils/subst.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textutils/subst.groovy b/textutils/subst.groovy index aae44c6..ea1678c 100644 --- a/textutils/subst.groovy +++ b/textutils/subst.groovy @@ -2,7 +2,7 @@ def substitute(String filepath, String pattern, String subst) { fileText = readFile(filepath) def converted = fileText.replaceAll(pattern, subst) - writeFile(file: filepath, text: converted + writeFile(file: filepath, text: converted) } return this; \ No newline at end of file From dad17a4f5a0016a5211858f84c9b3b39bfde8be5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 9 Aug 2019 14:52:51 +0100 Subject: [PATCH 589/589] GIS-7717 GIS-7718 - Text substitution correction --- textutils/subst.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/textutils/subst.groovy b/textutils/subst.groovy index ea1678c..055a269 100644 --- a/textutils/subst.groovy +++ b/textutils/subst.groovy @@ -1,8 +1,8 @@ def substitute(String filepath, String pattern, String subst) { - fileText = readFile(filepath) + fileText = readFile(file: filepath, encoding: 'UTF-8') def converted = fileText.replaceAll(pattern, subst) - writeFile(file: filepath, text: converted) + writeFile(file: filepath, text: converted, encoding: 'UTF-8') } return this; \ No newline at end of file