From 2dbfbdab472fc59f27e233fb3f9fdf5759a99278 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:14:13 +0000 Subject: [PATCH 1/8] Remove .git files/folders before publishing --- publish/do.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..91f3b3e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepo}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs rm -rvf + find . -name ".gitignore" | xargs rm -rvf + find . -name ".gitmodules" | xargs rm -rvf + find . -iname ".gitattributes" | xargs rm -rvf """ } } From 6ac6396c6f7ffeac25f73b7c8344d561b81e859d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:29:40 +0000 Subject: [PATCH 2/8] Remove .git files/folders before publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 91f3b3e..ec9b53f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,7 +90,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { - echo "Stripping git history out of ${releaserepo}" + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash pwd From 7a4c46d342ac3671f42d2077be48d0bbf21176ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 14:01:09 +0000 Subject: [PATCH 3/8] Releasing to git was relying on the source code sha changing only; However it needs to take into account that a dependent component revision number changed too --- release/do.groovy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } From ca914eebd209abcefaca8ffbbd1f21847e9de156 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:20:25 +0000 Subject: [PATCH 4/8] Remove .git detritus from published folders --- publish/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index ec9b53f..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -94,10 +94,10 @@ def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash pwd - find . -type d -name ".git" | xargs rm -rvf - find . -name ".gitignore" | xargs rm -rvf - find . -name ".gitmodules" | xargs rm -rvf - find . -iname ".gitattributes" | xargs rm -rvf + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } From 5a1a55aa35a0b9d2a4fae58c9c3143947aa65e68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:22:15 +0000 Subject: [PATCH 5/8] Move product agnostic scripts into common subtree area --- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From e00d396c2b5dab238d2b0bff57abf0a1d0a0c082 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 17:49:02 +0000 Subject: [PATCH 6/8] Remove redundant common_scripts --- file/utils.groovy | 16 --- versionnumbering/versioninfo.py | 180 -------------------------------- 2 files changed, 196 deletions(-) delete mode 100644 file/utils.groovy delete mode 100644 versionnumbering/versioninfo.py diff --git a/file/utils.groovy b/file/utils.groovy deleted file mode 100644 index 0ffa47c..0000000 --- a/file/utils.groovy +++ /dev/null @@ -1,16 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - -// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents -def currentContents(file, newline = "${nl}", defaultContents = "") { - def contents = "" - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - -return this; diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py deleted file mode 100644 index 3d2a80d..0000000 --- a/versionnumbering/versioninfo.py +++ /dev/null @@ -1,180 +0,0 @@ -# encoding: utf-8 -''' -versionnumbering -- shortdesc - -versionnumbering generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess -import re -import datetime - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - thirdpartyversionprefix = "1.25000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().find("master") != -1: # Special case - return mastertypeversionprefix - - splits2 = re.split(r'(\d+)', sourcebranch) - splits = [] - hasdot = False - for split in splits2: - if split.isdigit(): - splits.append(split) - if split == ".": - hasdot = True - - if not hasdot: - print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) - return thirdpartyversionprefix - - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - sourcebranch = args.sourcebranch - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - - print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber) - - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) From 1eba3bf0162fead6284b53e63647c2f2456326d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 18:00:34 +0000 Subject: [PATCH 7/8] Remove compile retries --- compile/eclipsebuild/eclipseds5build.groovy | 52 ++++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..cf382b1 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,39 +142,35 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ } } - else { - echo "Not building ${build_variant} variant of project: ${project}" + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } - } catch (Exception e) { - echo "Build failed - Waiting for a while" - sleep(time: 4, unit: 'SECONDS') - error "Build failed..." } + else { + echo "Not building ${build_variant} variant of project: ${project}" + } + } catch (Exception e) { + error "Build failed..." } } } From 93ef7dfcf78b80b3eb033320caa9494a0052cb33 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 12:05:47 +0000 Subject: [PATCH 8/8] Revert "Remove compile retries" This reverts commit 90ee2b5c1c5c2bcf13af8d3c501114f41b3d8181. --- compile/eclipsebuild/eclipseds5build.groovy | 52 +++++++++++---------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index cf382b1..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,35 +142,39 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } + } catch (Exception e) { + echo "Build failed - Waiting for a while" + sleep(time: 4, unit: 'SECONDS') + error "Build failed..." } - else { - echo "Not building ${build_variant} variant of project: ${project}" - } - } catch (Exception e) { - error "Build failed..." } } }