From e5bca0dbb6e629c137edd5325ec3e132c01f4696 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 15:46:25 +0000 Subject: [PATCH 01/72] Initial commit --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..d0349b6 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# release_info_autobuild + +release_info_autobuild release repo \ No newline at end of file From 9171d4d09383dd99d0fa5f06e3f4b50cbb26dca8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 17:01:43 +0000 Subject: [PATCH 02/72] Initial commit --- product/foreach.groovy | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 product/foreach.groovy diff --git a/product/foreach.groovy b/product/foreach.groovy new file mode 100644 index 0000000..690c5e1 --- /dev/null +++ b/product/foreach.groovy @@ -0,0 +1,61 @@ +def productcollectionstore = null + +/* Example expected JSON +{ + "ProductReleaseRepos": [ + "git@hydra:print-server-releases/print_server_whole.git" + ] +}*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: productcollectionstore, pretty: 1) +} + +def boolean exists() { + return (productcollectionstore != null) +} + +def set(String jsontext) { + productcollectionstore = readJSON(text: jsontext) + echo "Product collection:\n${asJsonText()}" +} + +def get() { + return productcollectionstore +} + +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + + set(PRODUCT_COLLECTION) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Iterate product collection') { + steps { + script { + def productcollection = get() + for (productrepo in productcollection.ProductReleaseRepos) { + } + } + } + } +} +} +PRODUCT_COLLECTION \ No newline at end of file From c1344e90845db3aa76de5f3d0fc93abb462f8782 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 17:04:54 +0000 Subject: [PATCH 03/72] Squashed 'Commons_scripts/' content from commit 0326879 git-subtree-dir: Commons_scripts git-subtree-split: 0326879391035840f42f5e1eb5ffc92e834c8995 --- .project | 17 + .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + artifacts/archiver.groovy | 17 + artifacts/metaexport.groovy | 26 + autobuilder.groovy | 147 ++ build/integrity.groovy | 113 ++ build/postactions.groovy | 225 ++++ build/postactionstest.groovy | 43 + check/build_meta.groovy | 87 ++ check/do_meta.groovy | 77 ++ checksum/utils.groovy | 15 + clean/clean.groovy | 10 + commandline/commandline.groovy | 12 + .../eclipsebuild/disableparallelbuilds.groovy | 12 + compile/eclipsebuild/eclipseSTM8build.groovy | 172 +++ compile/eclipsebuild/eclipseds5build.groovy | 252 ++++ compile/eclipsebuild/eclipseniosbuild.groovy | 116 ++ configuration/configuration.groovy | 18 + configuration/tools.groovy | 59 + copy/all.groovy | 67 + copy/esw.groovy | 67 + copy/filtered.groovy | 73 + copy/logic.groovy | 79 ++ differences/.project | 17 + differences/.pydevproject | 8 + differences/dometahierarchy.groovy | 836 ++++++++++++ differences/file.groovy | 24 + differences/json.groovy | 51 + differences/metahierarchy.groovy | 102 ++ documentation/doxygen_deploy.groovy | 80 ++ hardware/sominfo.groovy | 80 ++ import/importer.groovy | 408 ++++++ install/installhelper.groovy | 136 ++ meta/folder.groovy | 1193 +++++++++++++++++ packaging/debian.groovy | 111 ++ platform/lineendings.groovy | 5 + publish/.classpath | 9 + publish/.gitignore | 7 + publish/.project | 18 + .../org.eclipse.jdt.groovy.core.prefs | 3 + publish/do.groovy | 174 +++ publish/package.groovy | 181 +++ publish/promptingpublish.groovy | 182 +++ publish/publish.groovy | 92 ++ publish/publishitems.groovy | 129 ++ publishing/utils.groovy | 27 + release/do.groovy | 458 +++++++ release/registerrepo.groovy | 12 + release/release.groovy | 87 ++ remoteaccess/ftp.groovy | 32 + remoteaccess/hostedftp.groovy | 28 + staticanalysis/ccm.groovy | 85 ++ staticanalysis/cpd.groovy | 152 +++ staticanalysis/cppcheck.groovy | 114 ++ test/testethernetproduct.groovy | 204 +++ textutils/subst.groovy | 8 + .../vs2017install/buildmachines.vsconfig | 25 + versioncontrol/git.groovy | 377 ++++++ versionnumbering/do.groovy | 23 + versionnumbering/generate.groovy | 180 +++ versionstamping/.project | 17 + versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 38 + 64 files changed, 7435 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 artifacts/archiver.groovy create mode 100644 artifacts/metaexport.groovy create mode 100644 autobuilder.groovy create mode 100644 build/integrity.groovy create mode 100644 build/postactions.groovy create mode 100644 build/postactionstest.groovy create mode 100644 check/build_meta.groovy create mode 100644 check/do_meta.groovy create mode 100644 checksum/utils.groovy create mode 100644 clean/clean.groovy create mode 100644 commandline/commandline.groovy create mode 100644 compile/eclipsebuild/disableparallelbuilds.groovy create mode 100644 compile/eclipsebuild/eclipseSTM8build.groovy create mode 100644 compile/eclipsebuild/eclipseds5build.groovy create mode 100644 compile/eclipsebuild/eclipseniosbuild.groovy create mode 100644 configuration/configuration.groovy create mode 100644 configuration/tools.groovy create mode 100644 copy/all.groovy create mode 100644 copy/esw.groovy create mode 100644 copy/filtered.groovy create mode 100644 copy/logic.groovy create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/dometahierarchy.groovy create mode 100644 differences/file.groovy create mode 100644 differences/json.groovy create mode 100644 differences/metahierarchy.groovy create mode 100644 documentation/doxygen_deploy.groovy create mode 100644 hardware/sominfo.groovy create mode 100644 import/importer.groovy create mode 100644 install/installhelper.groovy create mode 100644 meta/folder.groovy create mode 100644 packaging/debian.groovy create mode 100644 platform/lineendings.groovy create mode 100644 publish/.classpath create mode 100644 publish/.gitignore create mode 100644 publish/.project create mode 100644 publish/.settings/org.eclipse.jdt.groovy.core.prefs create mode 100644 publish/do.groovy create mode 100644 publish/package.groovy create mode 100644 publish/promptingpublish.groovy create mode 100644 publish/publish.groovy create mode 100644 publish/publishitems.groovy create mode 100644 publishing/utils.groovy create mode 100644 release/do.groovy create mode 100644 release/registerrepo.groovy create mode 100644 release/release.groovy create mode 100644 remoteaccess/ftp.groovy create mode 100644 remoteaccess/hostedftp.groovy create mode 100644 staticanalysis/ccm.groovy create mode 100644 staticanalysis/cpd.groovy create mode 100644 staticanalysis/cppcheck.groovy create mode 100644 test/testethernetproduct.groovy create mode 100644 textutils/subst.groovy create mode 100644 tool configurations/vs2017install/buildmachines.vsconfig create mode 100644 versioncontrol/git.groovy create mode 100644 versionnumbering/do.groovy create mode 100644 versionnumbering/generate.groovy create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy new file mode 100644 index 0000000..bafba02 --- /dev/null +++ b/artifacts/archiver.groovy @@ -0,0 +1,17 @@ +enum Store { + JENKINS, + ARTIFACTSERVER, +} + +this.JENKINS = Store.JENKINS +this.ARTIFACTSERVER = Store.JENKINS + +def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { + echo "Archiving artifacts to store: ${store}" + meta.copyrelatedmeta() + if (store == JENKINS) { + archiveArtifacts(artifacts: artifactlist) + } +} + +return this \ No newline at end of file diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy new file mode 100644 index 0000000..a40e9a8 --- /dev/null +++ b/artifacts/metaexport.groovy @@ -0,0 +1,26 @@ +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(meta, importer, subJobs) { + def ArrayList artifactFolders = new ArrayList() + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getFullProjectName() + def subJobNumber = subJobObject.getNumber() + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] + dir(destFolder) { + deleteDir() + } + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(subJobName, subJobNumber) + importoptions.setincrementversionnumber(incrementversionnumber) + importer.artifactimport(meta, importoptions, destFolder) + + artifactFolders.add("${destFolder}/**") + } + } + return artifactFolders +} + +return this; diff --git a/autobuilder.groovy b/autobuilder.groovy new file mode 100644 index 0000000..667f89a --- /dev/null +++ b/autobuilder.groovy @@ -0,0 +1,147 @@ +// Assume that WORKSPACE environment variable is set + +autobuildroot = 'autobuild' +commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + def stashname = 'Clean workspace stash' + stash(allowEmpty: true, + includes: "${autobuildroot}/**,${autobuildroot}/.git/**", + name: stashname, + useDefaultExcludes: false + ) + cleanWs() + unstash name: stashname + } + else { + echo "Not cleaning workspace" + } +} + +def commonscripts() { + return commonscriptsroot +} + +def getautobuildpath() { + return autobuildroot +} + +def getmetafolderscript() { + return metafolderscript +} + +def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { + def String altered = releaserepo.trim() + + if (releaserepovariantsuffix?.trim()) { + altered = altered.replace("-RELEASE_REPO_VARIANT_SUFFIX-", releaserepovariantsuffix) + } + return altered +} + +def setBuildType(meta, boolean dofullbuild = true) { + def doquickbuild = false + + echo "Build trigger cause: ${currentBuild.getBuildCauses()}" + if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { + echo "Build triggered by user" + if (!dofullbuild) { + echo "Marking as quick build as user wanted one" + doquickbuild = true + } + } else { + echo "Build triggered automatically" + if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { + echo "Marking as quick build as it was triggered by a commit" + doquickbuild = true + } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { + echo "Marking as full build as it was triggered by timer" + } + } + echo "Build type is ${doquickbuild ? "quick" : "full"}" + if (doquickbuild) { + unstable('This not a full build - a subset of the artifacts is not being generated') + } + meta.jenkinsbuildinfo.set(!doquickbuild) + return doquickbuild +} + +def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { + def meta = getmetafolderscript().metareset(script) + echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" + meta.productinfo.set(productname, subcomponentname, productvariant) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + for (p in params) { + meta.jenkinsbuildinfo.addParameter(p.key, p.value) + } + return meta +} + +// Start up boilerplate +def startbuild( + script, + String productname = PRODUCT_NAME, + String subcomponentname = SUBCOMPONENT_NAME, + String releaserepo = RELEASE_REPO, + String productvariant = '', + String autobuildpath = autobuildroot, + String commonscriptspath = commonscriptsroot + ) { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = setupmeta(script, productname, subcomponentname, productvariant) + + def rr = load("${commonscripts()}/release/registerrepo.groovy") + def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) + + echo "Release repo: ${alteredreleaserepo}" + rr.registerrepo(meta, alteredreleaserepo) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +// Start up boilerplate +def startnonbuild( + script, + autobuildpath = autobuildroot, + commonscriptspath = commonscriptsroot + ) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.groovy") +} + +// Everyone wants to know this +autobuilder = this + +return this; diff --git a/build/integrity.groovy b/build/integrity.groovy new file mode 100644 index 0000000..e93c00f --- /dev/null +++ b/build/integrity.groovy @@ -0,0 +1,113 @@ +import static groovy.json.JsonOutput.* + +def check(buildmeta) { + def results = [ + 'good': true, + 'comment': '', + ] + + // Walk the meta looking for different versions of the same components + def allmeta = buildmeta.allrelatedmeta() + echo "All meta:\n${allmeta.comment}" + + // Store build stamps associated with each product component; Store a counter against each product component version + def productcomponents = [:] + for (meta in allmeta.metacollection) { + def key = meta.componentidentifier() + + if (!productcomponents.containsKey(key)) { + productcomponents[key] = [:] + } + def buildstampnumber = meta.releasetag() + if (!productcomponents[key].containsKey(buildstampnumber)) { + productcomponents[key][buildstampnumber] = 1 + } else { + productcomponents[key][buildstampnumber] += 1 + } + } + + // Show what was collected + echo prettyPrint(toJson(productcomponents)) + + // Look for product components with more than one build stamp + for (pc in productcomponents.keySet()) { + def bsns = productcomponents[pc] + if (bsns.size() > 1) { + results.good = false + results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" + for (bsn in bsns.keySet()) { + results.comment += "${bsn}\n" + } + } + } + + // Store product components associated with each release git url + def releasegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.releaseinfo.releaserepoURL + + if (key) { + if (!releasegiturls.containsKey(key)) { + releasegiturls[key] = [:] + } + def componentidentifier = meta.componentidentifier() + if (!releasegiturls[key].containsKey(componentidentifier)) { + releasegiturls[key][componentidentifier] = 1 + } else { + releasegiturls[key][componentidentifier] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(releasegiturls)) + + // Look for release git urls being used by more than one product component + for (rgu in releasegiturls.keySet()) { + def cis = releasegiturls[rgu] + if (cis.size() > 1) { + results.good = false + results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" + for (ci in cis.keySet()) { + results.comment += "${ci}\n" + } + } + } + + // Store git commit shas associated with each source git url + def sourcegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.gitinfo.remoteurl + + if (key) { + if (!sourcegiturls.containsKey(key)) { + sourcegiturls[key] = [:] + } + def commitsha = meta.gitinfo.gitrevision + if (!sourcegiturls[key].containsKey(commitsha)) { + sourcegiturls[key][commitsha] = 1 + } else { + sourcegiturls[key][commitsha] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(sourcegiturls)) + + // Look for source git urls associated with more than one sha code + for (sgu in sourcegiturls.keySet()) { + def cs = sourcegiturls[sgu] + if (cs.size() > 1) { + results.good = false + results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" + for (c in cs.keySet()) { + results.comment += "${c}\n" + } + } + } + + return results +} + +return this \ No newline at end of file diff --git a/build/postactions.groovy b/build/postactions.groovy new file mode 100644 index 0000000..8b18908 --- /dev/null +++ b/build/postactions.groovy @@ -0,0 +1,225 @@ +/* Example post build info +{ + "Build outcomes": { + "Success": { + "Actions": { + "Deploy documentation": { + "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" + }, + "Static analysis": { + "Description": "Can be empty - Static analysis job to run when this build is successful", + "Job": "" + }, + "Unit test": { + "Description": "Can be empty - Unit test job to run when this build is successful", + "Job": "" + }, + "Regression test": { + "Description": "Can be empty - Regression test job to run when this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" + }, + "Update differences": { + "Description": "Can be empty - Update differences job to run when this build is successful", + "Job": "HMB_FD_HV_diff_releases" + }, + "Release to git": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "Job": "HMB_FD_HV_release_to_git" + } + } + }, + "Failed": { + "Actions": { + "Email": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ] + }, + "ExtEmail": { + "Description": "Send an email using Extended Email plugin", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ], + "Recipient providers": [ + "culprits", + "requestor", + "developers", + "upstreamDevelopers" + ] + } + } + } + } +} +*/ + +def getActionJobName(postbuildactions, String action, String alternativeparamname) { + def String jobname = '' + if (postbuildactions) { + try { + jobname = postbuildactions[action]['Job'] + } catch(Exception ex) { + echo "No post build actions found for action: ${action}" + jobname = '' + } + } else { + if (params[alternativeparamname] != null) { + if (params[alternativeparamname]) { + jobname = params[alternativeparamname] + } + } + } + if (jobname) { + def jobitem = Jenkins.instance.getItemByFullName(jobname) + if (jobitem) { + if (!jobitem.isBuildable()) { + jobname = '' + } + } else { + jobname = '' + } + } + // echo "Job name for action: ${action}: ${jobname}" + return jobname +} + +def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { + def jobname = getActionJobName(postbuildactions, action, alternativeparamname) + if (jobname) { + build( + wait: false, + job: jobname, + parameters: [ + string(name: jobnameparametername, value: JOB_NAME), + string(name: jobselectorparametername, value: buildselector), + ] + ) + } +} + +def readPostBuildInfo() { + def postbuildinfo = readJSON(text: POST_BUILD_INFO) + def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) + // echo "Post build info:\n${postbuildjsontext}" + + return postbuildinfo +} + +def performSuccessfulBuildActions() { + def postbuildactions + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Success/Actions" + postbuildactions = null + } + } + def String buildselector = "${BUILD_NUMBER}" + deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') + deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') +} + +def sendemails(emailinfo) { + def emailtolist = emailinfo['To'] + + if (emailtolist) { + def emailbody = """ + Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} + + Console output: ${BUILD_URL}console + """ + for(emailto in emailtolist) { + mail( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: emailbody, + to: emailto + ) + // echo "Send email to: ${emailto}" + } + } +} + +def getRecipientProviders(recipientProviderInfoList) { + def recipientProviderMap = [ + culprits: [$class: 'CulpritsRecipientProvider'], + requestor: [$class: 'RequesterRecipientProvider'], + developers: [$class: 'DevelopersRecipientProvider'], + upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], + ] + def recipientProviders = [] + + if (recipientProviderInfoList) { + for(rp in recipientProviderInfoList) { + if (recipientProviderMap.containsKey(rp)) { + recipientProviders.add(recipientProviderMap[rp]) + } + } + } + if (recipientProviders.size() == 0) { + recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] + } + // echo "recipientProviders: ${recipientProviders.toString()}" + return recipientProviders; +} + +def String getToList(toListInfo) { + def String toList = '' + + if (toListInfo) { + toList = toListInfo.join(',') + } + // echo "toList: ${toList}" + return toList +} + +def sendextemail(extmailinfo) { + def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) + def toList = getToList(extmailinfo['To']) + + emailext( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: '''${JELLY_SCRIPT,template="html"}''', + attachLog: true, + // attachmentsPattern: '', + compressLog: false, + // from : '', + mimeType: 'text/html', + // postsendScript: '', + // presendScript: '', + recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers + // replyTo: '', + saveOutput: false, + to: toList, + ) +} + +def performFailedBuildActions() { + def postbuildactions = null + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Failed/Actions" + postbuildactions = null + } + } + if (postbuildactions) { + if (postbuildactions['Email']) { + sendemails(postbuildactions['Email']) + } + if (postbuildactions['ExtEmail']) { + sendextemail(postbuildactions['ExtEmail']) + } + } +} + +return this; diff --git a/build/postactionstest.groovy b/build/postactionstest.groovy new file mode 100644 index 0000000..150f5d1 --- /dev/null +++ b/build/postactionstest.groovy @@ -0,0 +1,43 @@ +pipeline { + agent { label 'master' } + stages { + stage('Initial setup') { + steps { + script { + CLEAN_WORKSPACE = false + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Dummy build job') { + steps { + script { + echo "Execute dummy build job: ${CAUSE_ERROR}" + if (CAUSE_ERROR == 'true') { + error("Causing error") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() + } + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() + } + } + } +} diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..fb8a75d --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..4b8a89e --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,77 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = entry.metainfo.componentversionuniqueidentifier() + def offsetpath = entry.accumulatedpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + + // Walk the meta information to a max depth + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.accumulatedpath + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..be7569d --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,15 @@ + +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(meta, checksummable) { + for (filepath in checksummable) { + def checksumset = meta.checksuminfo.addChecksumSet(filepath) + def files = findFiles(glob: filepath) + for (file in files) { + echo "Checksumming: ${file.path}" + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) + } + } +} + +return this; \ No newline at end of file diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file diff --git a/commandline/commandline.groovy b/commandline/commandline.groovy new file mode 100644 index 0000000..38f80fa --- /dev/null +++ b/commandline/commandline.groovy @@ -0,0 +1,12 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return powershell(returnStdout: true, script: """ + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.groovy b/compile/eclipsebuild/disableparallelbuilds.groovy new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.groovy @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + + return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/configuration/tools.groovy b/configuration/tools.groovy new file mode 100644 index 0000000..dbaeedb --- /dev/null +++ b/configuration/tools.groovy @@ -0,0 +1,59 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +class ToolOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + String m_toolName + + String m_gitURL + String m_revision + String m_repoRoot + int m_cloneDepth + boolean m_shallowClone + + ToolOptions(script) { + m_script = script + m_gitURL = '-' + m_revision = '-' + m_repoRoot = '' + m_cloneDepth = 1 + m_shallowClone = true + } + + void toolName(String name) { + m_toolName = name.trim() + } + + void toolRepo(String gitURL, String revision) { + m_gitURL = gitURL.trim() + m_revision = revision.trim() + } + + void repoRoot(String root) { + m_repoRoot = root.trim() + } + + void clonedepth(int depth) { + m_cloneDepth = depth + } + + void shallowclone(boolean shallow) { + m_shallowClone = shallow + } +} + +ToolOptions getNewToolOptions() { + return new ToolOptions(this) +} + +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) +} + +def getTool(meta, ToolOptions options, String targetDir) { + gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) + + registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/copy/all.groovy b/copy/all.groovy new file mode 100644 index 0000000..24aacfb --- /dev/null +++ b/copy/all.groovy @@ -0,0 +1,67 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + // Force workspace to be cleaned + CLEAN_WORKSPACE = "true" + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + } + } + } + stage('Import source artifacts') { + steps { + script { + def copyoptions = importer.getNewImportOptions() + copyoptions.source( + SOURCE_ARTIFACT_LOCATION, + SOURCE_ARTIFACT_BUILD_JOB, + SOURCE_ARTIFACT_BUILD_SELECTOR, + SOURCE_ARTIFACT_RELEASE_REPO, + SOURCE_ARTIFACT_RELEASE_REVISION) + echo "Nested: ${NESTED_COPY}" + copyoptions.setnested(NESTED_COPY == 'true') + importer.artifactimport(meta, copyoptions, "${WORKSPACE}/.") + } + } + } + stage('Archiving artifacts') { + steps { + script { + // Save the imported meta - while the unwanted meta is written to file then deleted + def stashname = 'Imported meta stash' + stash(allowEmpty: true, + includes: 'meta/**', + name: stashname, + useDefaultExcludes: false) + + def artifactlist = meta.generateartifacts(['**/*']) + cleaner.clean(['meta', 'autobuild']) + + unstash(name: stashname) + + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/copy/esw.groovy b/copy/esw.groovy new file mode 100644 index 0000000..b9eb737 --- /dev/null +++ b/copy/esw.groovy @@ -0,0 +1,67 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + env.ESW = 'esw' + env.ESWDEST = "${WORKSPACE}/${ESW}" + env.ESWEXPORT = 'embedded_software' + cleanfolders = [ + ESWDEST, + ESWEXPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying embedded software artifacts') { + steps { + script { + def eswimportoptions = importer.getNewImportOptions() + eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) + eswimportoptions.setincrementversionnumber(true) + importer.artifactimport(meta, eswimportoptions, ESWDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) + sh """ + rm -v -rf diff + rm -v -rf meta + mv -v "${ESW}"/* . + """ + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} \ No newline at end of file diff --git a/copy/filtered.groovy b/copy/filtered.groovy new file mode 100644 index 0000000..3ad24cd --- /dev/null +++ b/copy/filtered.groovy @@ -0,0 +1,73 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean') { + steps { + script { + env.IMPORT = "${WORKSPACE}/import" + env.FILTEREDDEST = "${WORKSPACE}/filtered" + cleanfolders = [ + FILTEREDDEST, + IMPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import artifacts') { + steps { + script { + def importoptions = importer.getNewImportOptions() + importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) + importer.artifactimport(meta, importoptions, IMPORT) + } + } + } + stage('Filtered copy') { + steps { + script { + dir(IMPORT) { + fileOperations([ + fileCopyOperation(excludes: EXCLUDES, + flattenFiles: FLATTENOUTPUT, + includes: INCLUDES, + targetLocation: FILTEREDDEST)]) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["filtered/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..db9dbc3 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,79 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + LOGICDEST = 'logic' + cleanfolders = [ + LOGICDEST, + "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/el", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import source artifacts') { + steps { + script { + def logicoptions = importer.getNewImportOptions() + logicoptions.source( + LOGIC_ARTIFACT_LOCATION, + LOGIC_ARTIFACT_PROJECT, + LOGIC_ARTIFACT_BUILD_SELECTOR, + LOGIC_ARTIFACT_RELEASE_REPO, + LOGIC_ARTIFACT_RELEASE_REVISION) + importer.artifactimport(meta, logicoptions, LOGICDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + + dir(LOGICDEST) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + + cleaner.clean([LOGICDEST]) + + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/differences/.project b/differences/.project new file mode 100644 index 0000000..75af0fc --- /dev/null +++ b/differences/.project @@ -0,0 +1,17 @@ + + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy new file mode 100644 index 0000000..3b25798 --- /dev/null +++ b/differences/dometahierarchy.groovy @@ -0,0 +1,836 @@ +import groovy.json.JsonSlurper + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" + +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" + +reportfolder = "${WORKSPACE}/reports" + +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } +} + +class DifferenceRecord { + String productname + String subcomponentname + + String rawdifferencetext + String sha + String type + String mergeinfo + String squashinfo + List jiraticketinfo + String comment + + DifferenceRecord() { + productname = "" + subcomponentname = "" + + rawdifferencetext = "" + sha = "" + type = "" + mergeinfo = "" + squashinfo = "" + jiraticketinfo = [] + comment = "" + } + + def String dump() { + String jiraticketstring = '' + for (j in jiraticketinfo) { + jiraticketstring += j.dump() + " " + } + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + } +} + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } + dir(reportfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def showmetaitems(metaitems) { + String info = '' + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + def accumulatedpath = relatedmetainfo.accumulatedpath + info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" + } + echo info +} + +def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { + boolean checkedout = false + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo?.trim()) { + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } + checkedout = true + } + return checkedout +} + +def String getcomponent(meta) { + def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) + + return [checkedout, uniquefolderforsourcerepo] +} + +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def String releaseexpression + if (tripledot) { + releaseexpression = earlyrevision + "..." + laterrevision + } else { + releaseexpression = earlyrevision + ".." + laterrevision + } + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() + } + return differencetext +} + +def extractsha(String line) { + def pattern = ~"^(\\p{XDigit}*) (.*)" + def match = (line =~ pattern) + match.find() + + def sha = match.group(1) + def restofline = match.group(2) + + //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" + + return [sha.trim(), restofline.trim()] +} + +def extractsquashes(String line) { + def pattern1 = "^(.*)(Squashed .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + return [false, line] +} + +def extractmerges(String line) { + def pattern1 = "^(.*)(Merge branch .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + def pattern2 = "^(.*)(Merge remote-tracking branch .*)" + def match2 = (line =~ pattern2) + if (match2.find()) { + return [true, match2.group(2)] + } + + def pattern3 = "^(.*)(Merge commit .*)" + def match3 = (line =~ pattern3) + if (match3.find()) { + return [true, match3.group(2)] + } + + def pattern4 = "^(.*)(Add .* from commit .*)" + def match4 = (line =~ pattern4) + if (match4.find()) { + return [true, match4.group(2)] + } + + return [false, line] +} + +def jiraticketexists(String jiraticket) { + def String URI = "${JIRA_URI}/browse/${jiraticket}" + def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" + def get = new URL(RestfulURI).openConnection(); + get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) + get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) + get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) + get.setRequestProperty( 'Accept', 'application/json' ) + def getRC = get.getResponseCode(); + + def String redirectedjiraticket = jiraticket + if(!getRC.equals(200)) { + return [false, '', jiraticket] + } else { + def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) + redirectedjiraticket = jsonresponse['key'] + } + return [true, URI, redirectedjiraticket] +} + +def extractjiraticketinfo(String line) { + def List jiratickets = [] + boolean exists = false + String URI = '' + + while (true) { + line = line.trim() + def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line + def commamatch = (line =~ commapattern) + if (commamatch.find()) { + line = commamatch.group(1).trim() + } + def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def match1 = (line =~ pattern1) + if (match1.find()) { + jiraticket = match1.group(1) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match1.group(2) + } else { + break + } + } else { + def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 + def match2 = (line =~ pattern2) + if (match2.find()) { + jiraticket = match2.group(1) + "-" + match2.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match2.group(3) + } else { + break + } + } else { + def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 + def match3 = (line =~ pattern3) + if (match3.find()) { + jiraticket = match3.group(1) + "-" + match3.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match3.group(3) + } else { + break + } + } else { + break + } + } + } + } + boolean result = jiratickets.size() > 0 + return [result, jiratickets, line] +} + +def extractreverts(String line) { + boolean result = true + + def pattern1 = '^(.*)Revert \"(.*)\"' + def match1 = (line =~ pattern1) + if (match1.find()) { + def line2 = match1.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + def pattern2 = '^(.*)Partial Revert \"(.*)\"' + def match2 = (line =~ pattern2) + if (match2.find()) { + def line2 = match2.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + return [false, [], line] +} + +def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() + } + return differencetext +} + +def List fillsquashrecords(String sourcepath, String sha) { + def List differencerecords = [] + + String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) + int count = 0 + squashloglines.split('\n').each { + if (count > 0) { // Ignore first line + def line = it + if (line?.trim()) { + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) + //echo "${dr.dump()}" + } + } + count += 1 + } + return differencerecords +} + +def List fillrecord(meta, line, String sourcepath) { + def List differencerecords = [] + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + + dr.rawdifferencetext = line + + // Example of raw text to process: + // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py + + (sha, line2) = extractsha(line) + dr.sha = sha + + (result, squashinfo) = extractsquashes(line2) + if (result) { + dr.type = "Squash" + dr.squashinfo = squashinfo + squashrecords = fillsquashrecords(sourcepath, sha) + differencerecords.addAll(squashrecords) + } else { + (result, mergeinfo) = extractmerges(line2) + if (result) { + dr.type = "Merge" + dr.mergeinfo = mergeinfo + } else { + (result, jiraticketinfo, comment) = extractreverts(line2) + if (result) { + dr.type = "Revert" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.comment = line2 + } + } + } + } + differencerecords.add(dr) + return differencerecords +} + +def List processtext(meta, String text, String sourcepath) { + def List differencerecords = [] + + text.split('\n').each { + def line = it + if (line?.trim()) { + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) + //echo "${dr.dump()}" + } + } + return differencerecords +} + +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + differencerecords = processtext(earlymeta, differencetext, sourcepath) + if (earlyrevision == laterrevision) { + summaryappend(" No change") + } + } else { + summaryappend("The early and later git urls don't match") + echo "The early and later git urls don't match" + result = false; + setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") + } + + return [result, differencerecords] +} + +def deletedcomponent(earlymetaitems, key) { + echo "Component ${key} has been deleted in later version" + summaryappend("Deleted ") +} + +def addedcomponent(latermetaitems, key) { + echo "Component ${key} has been added in later version" + summaryappend("Added ") +} + +class SquashReportRecord { + String comment + int duplicates + + SquashReportRecord() { + comment = '' + duplicates = 0 + } +} + +def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { + dir(reportsubfolder) { + def String squashlist = "${title}\nSquash list:\n" + def squashessorted = squashes.sort() + for (asquash in squashessorted) { + def SquashReportRecord s = asquash.value + squashlist += "${s.comment} (${s.duplicates})\n" + } + writeFile(file: "squashlist.txt", text: squashlist) + } +} + +class MergeReportRecord { + String comment + int duplicates + + MergeReportRecord() { + comment = '' + duplicates = 0 + } +} + +def mergereport(String reportsubfolder, String title, LinkedHashMap merges, boolean iscomponent = true) { + dir(reportsubfolder) { + def String mergelist = "${title}\nMerge list:\n" + def mergessorted = merges.sort() + for (amerge in mergessorted) { + def MergeReportRecord m = amerge.value + mergelist += "${m.comment} (${m.duplicates})\n" + } + writeFile(file: "mergelist.txt", text: mergelist) + } +} + +class CommentsRecord { + String comment + int duplicates + + CommentsRecord() { + comment = '' + duplicates = 0 + } +} + +class RevertsReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + RevertsReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def revertsreport(String reportsubfolder, String title, LinkedHashMap reverts, boolean iscomponent = true) { + dir(reportsubfolder) { + def String revertlist = "${title}\nRevert list:\n" + def revertssorted = reverts.sort() + for (arevert in revertssorted) { + def RevertsReportRecord r = arevert.value + revertlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + revertlist += "${c.comment} (${c.duplicates})\n" + } + revertlist += "\n" + } + writeFile(file: "revertlist.txt", text: revertlist) + } +} + +class TicketedReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + TicketedReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def ticketedreport(String reportsubfolder, String title, LinkedHashMap ticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String ticketedlist = "${title}\nTicketed list:\n" + def ticketedsorted = ticketed.sort() + for (aticketed in ticketedsorted) { + def TicketedReportRecord r = aticketed.value + ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + ticketedlist += "${c.comment} (${c.duplicates})\n" + } + ticketedlist += "\n" + } + writeFile(file: "ticketedlist.txt", text: ticketedlist) + } +} + +class UnticketedReportRecord { + String comment + int duplicates + + UnticketedReportRecord() { + comment = '' + duplicates = 0 + } +} + +def unticketedreport(String reportsubfolder, String title, LinkedHashMap unticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String unticketedlist = "${title}\nUnticketed list:\n" + def unticketedsorted = unticketed.sort() + for (anunticketed in unticketedsorted) { + UnticketedReportRecord u = anunticketed.value + unticketedlist += "${u.comment} (${u.duplicates})\n" + } + writeFile(file: "unticketedlist.txt", text: unticketedlist) + } +} + +class TicketsReportRecord { + String jiraticket + String ticketURI + int duplicates + + TicketsReportRecord() { + jiraticket = '' + ticketURI = '' + duplicates = 0 + } +} + +def ticketsreport(String reportsubfolder, String title, tickets, boolean iscomponent = true) { + dir(reportsubfolder) { + String ticketlist = "${title}\nTicket list:\n" + ticketssorted = tickets.sort() + for (ticketnumber in ticketssorted) { + TicketsReportRecord t = ticketnumber.value + ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" + } + writeFile(file: "ticketlist.txt", text: ticketlist) + } +} + +def differencerecordsreport(String reportsubfolder, String title, differencerecords, boolean iscomponent = true) { + dir(reportsubfolder) { + String recordlist = "${title}\n" + for (dr in differencerecords) { + recordlist += dr.dump() + "\n\n" + } + writeFile(file: "recordlist.txt", text: recordlist) + } +} + +def boolean report(meta, title, differencerecords, boolean iscomponent = true) { + def Map ticketed = [:] + def Map squashes = [:] + def Map merges = [:] + def Map reverts = [:] + def Map unticketed = [:] + def Map tickets = [:] + + for (dr in differencerecords) { + if (dr.type == "Squash") { + def SquashReportRecord s + if (!squashes.containsKey(dr.mergeinfo)) { + s = new SquashReportRecord() + s.comment = dr.squashinfo + s.duplicates = 1 + } else { + s = squashes[dr.squashinfo] + s.duplicates += 1 + } + squashes[dr.squashinfo] = s + } + if (dr.type == "Merge") { + def MergeReportRecord m + if (!merges.containsKey(dr.mergeinfo)) { + m = new MergeReportRecord() + m.comment = dr.mergeinfo + m.duplicates = 1 + } else { + m = merges[dr.mergeinfo] + m.duplicates += 1 + } + merges[dr.mergeinfo] = m + } + if (dr.type == "Revert") { + def RevertsReportRecord r + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!reverts.containsKey(j.jiranumber)) { + r = new RevertsReportRecord() + r.jiraticket = j.jiranumber + r.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + r = reverts[j.jiranumber] + def CommentsRecord c + if (!r.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + c = r.commentscollection[dr.comment] + c.duplicates += 1 + } + r.commentscollection[dr.comment] = c + } + reverts[j.jiranumber] = r + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 + } else { + t = tickets[j.jiranumber] + t.duplicates += 1 + } + tickets[j.jiranumber] = t + } + } + if (dr.type == "Ticketed") { + def TicketedReportRecord t + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!ticketed.containsKey(j.jiranumber)) { + t = new TicketedReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + t = ticketed[j.jiranumber] + def CommentsRecord c + if (!t.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + c = t.commentscollection[dr.comment] + c.duplicates += 1 + } + t.commentscollection[dr.comment] = c + } + ticketed[j.jiranumber] = t + def TicketsReportRecord ti + if (!tickets.containsKey(j.jiranumber)) { + ti = new TicketsReportRecord() + ti.jiraticket = j.jiranumber + ti.ticketURI = j.uri + ti.duplicates = 1 + } else { + ti = tickets[j.jiranumber] + ti.duplicates += 1 + } + tickets[j.jiranumber] = ti + } + } + if (dr.type == "Unticketed") { + UnticketedReportRecord u + if (!unticketed.containsKey(dr.comment)) { + u = new UnticketedReportRecord() + u.comment = dr.comment + u.duplicates = 1 + } else { + u = unticketed[dr.comment] + u.duplicates += 1 + } + unticketed[dr.comment] = u + } + } + String reportsubfolder = "" + if (iscomponent) { + reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" + } else { + reportsubfolder = "${reportfolder}/Combined" + } + mergereport(reportsubfolder, title, merges, iscomponent) + squashreport(reportsubfolder, title, squashes, iscomponent) + revertsreport(reportsubfolder, title, reverts, iscomponent) + ticketedreport(reportsubfolder, title, ticketed, iscomponent) + unticketedreport(reportsubfolder, title, unticketed, iscomponent) + ticketsreport(reportsubfolder, title, tickets, iscomponent) + differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) +} + +def title(earlymeta, latermeta) { + return """ + Differences in: + Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} + between: + then: + Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} + now: + Build date: ${latermeta.jenkinsbuildinfo.builddate} Build stamp: ${latermeta.buildstampinfo.buildstamp} Git revision: ${latermeta.gitinfo.gitrevision} + """ +} + +def compare(earlymetaitems, latermetaitems, key) { + boolean result = true; + echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].metainfo + def latermeta = latermetaitems[key].metainfo + + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + + def boolean repoavailable + def String uniquefolderforsourcerepo + (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) + + List differencerecords = [] + if (repoavailable) { + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + summaryappend(" Items: ${differencerecords.size()} ") + report(earlymeta, title(earlymeta, latermeta), differencerecords) + } else { + summaryappend(" No source repo ") + } + summaryappend("\n") + + return [result, differencerecords]; +} + +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def List differencerecords = [] + + def int md = maxdepth.toInteger() + resetfolders() + preconditions() + + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + + def earlymeta = meta.importrelated(true, earlymetafolder) + def latermeta = meta.importrelated(true, latermetafolder) + + def earlymetaitems = earlymeta.releasewalkerunique(md, '', earlymetafolder) + def latermetaitems = latermeta.releasewalkerunique(md, '', latermetafolder) + + echo "Early items:" + showmetaitems(earlymetaitems) + + echo "Later items:" + showmetaitems(latermetaitems) + + for (key in earlymetaitems.keySet()) { + if (latermetaitems.containsKey(key)) { + (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + differencerecords = differencerecords + componentdifferencerecords + } else { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + deletedcomponent(earlymetaitems, key) + summaryappend("\n") + } + } + for (key in latermetaitems.keySet()) { + if (!earlymetaitems.containsKey(key)) { + summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + addedcomponent(latermetaitems, key) + summaryappend("\n") + } + } + report(meta, title(earlymeta, latermeta), differencerecords, false) + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..ee1b8b3 --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,24 @@ +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { + echo "Comparing file: ${importedfile} with: ${currentfile} option: ${ignorewhitespace}" + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } + if (isUnix()) { + sh """ + diff -c ${wsparameter} "${importedfile}" "${currentfile}" + """ + } + else { + // diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # where diff + # if errorlevel 1 EXIT /b 1 + + diff.exe -c ${wsparameter} "${importedfile}" "${currentfile}" + """ + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy new file mode 100644 index 0000000..dbf47b8 --- /dev/null +++ b/differences/json.groovy @@ -0,0 +1,51 @@ +def comparefiles(importedjsonfile, currentjsonfile) { + echo "Comparing json file: ${importedjsonfile} with: ${currentjsonfile}" + if (isUnix()) { + sh """ + which jq + if [ \$? -ne 0 ]; then exit 1; fi + # jq -S sorts json output + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") + """ + } + else { + // Jq and diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # set + # echo %PATH% + try { + if (Get-Command jq) {} + } + catch { + echo "jq is unavailable" + exit \$false + } + try { + if (Get-Command diff) {} + } + catch { + echo "diff is unavailable" + exit \$false + } + + # echo Sorting json in files to compare + # jq -S sorts json output + jq -S . "${importedjsonfile}" > temp1.txt + if (\$lastexitcode -ge 1) { exit 1 } + jq -S . "${currentjsonfile}" > temp2.txt + if (\$lastexitcode -ge 1) { exit 1 } + + # echo Diffing sorted json files + diff.exe -c temp1.txt temp2.txt + \$lec = \$lastexitcode + + Remove-Item temp1.txt + Remove-Item temp2.txt + + if (\$lec -ge 1) { exit 1 } + """ + } +} + +return this; diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy new file mode 100644 index 0000000..01c1890 --- /dev/null +++ b/differences/metahierarchy.groovy @@ -0,0 +1,102 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + SUMMARYDIR = "${WORKSPACE}/summary" + REPORTDIR = "${WORKSPACE}/reports" + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + "${EARLYDIR}", + "${LATERDIR}", + "${SUMMARYDIR}", + "${REPORTDIR}", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Obtain top level meta') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() + + def earlyimportoptions = importer.getNewImportOptions() + earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) + earlyimportoptions.setincrementversionnumber(false) + earlyimportoptions.setfilter('meta/**') + importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) + + def laterimportoptions = importer.getNewImportOptions() + laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) + laterimportoptions.setincrementversionnumber(false) + laterimportoptions.setfilter('meta/**') + importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) + } + } + } + stage('Diff components') { + steps { + script { + def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,reports/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/documentation/doxygen_deploy.groovy b/documentation/doxygen_deploy.groovy new file mode 100644 index 0000000..5054c02 --- /dev/null +++ b/documentation/doxygen_deploy.groovy @@ -0,0 +1,80 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean imports') { + steps { + script { + env.DOXYGEN = "doxygen" + cleanfolders = [ + DOXYGEN, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import doxygen artifacts') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + def importoptions = importer.getNewImportOptions() + + importoptions.jenkinsjob(BUILD_JOB_WITH_DOXYGEN_ARTIFACTS, BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR) + importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') + doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) + + productname = doxygenmeta.productinfo.productname + buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix + + echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}" + } + } + } + stage('Copy doxygen to server') { + steps { + script { + doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" + pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}" + echo "Copying from ${doxygensource} to ${pathonserver}" + if (isUnix()) { + sh """#!/bin/bash + rm -rfv "${pathonserver}" + if [ \$? -ne 0 ]; then exit 1; fi + + mkdir -pv "${pathonserver}" + if [ \$? -ne 0 ]; then exit 1; fi + + echo mv -v "${doxygensource}"/ "${pathonserver}" + mv -v "${doxygensource}"/ "${pathonserver}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + error("This job does not run on Windows at the moment") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/hardware/sominfo.groovy b/hardware/sominfo.groovy new file mode 100644 index 0000000..72a3ca9 --- /dev/null +++ b/hardware/sominfo.groovy @@ -0,0 +1,80 @@ +def sominfostore = null + +/* Example expected JSON +{ + "SomType": { + "Som_BA4": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", + "Release repo revision": "" + } + }, + "Som_FC5": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", + "Release repo revision": "" + } + } + } +} +*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: sominfostore, pretty: 1) +} + +def boolean exists() { + return (sominfostore != null) +} + +def set(String jsontext) { + sominfostore = readJSON(text: jsontext) + echo "Som support info:\n${asJsonText()}" +} + +def get() { + return sominfostore +} + +def somtypes() { + return sominfostore['SomType'] +} + +def artifactinfo(somtype) { + return somtype['Artifact info'] +} + +def artifactlocation(artifactinfo) { + return artifactinfo['Location'] +} + +def boolean isjoblocation(artifactinfo) { + return (artifactinfo['Location'] == 'Jenkins job artifact' || + artifactinfo['Location'] == 'Jenkins job artefact') +} + +def boolean isreleaserepolocation(artifactinfo) { + return artifactinfo['Location'] == 'Release repository' +} + +def artifactjob(artifactinfo) { + return [ + "name": artifactinfo['Job'], + "selector": "${artifactinfo['Job selector']}" + ] +} + +def releaserepo(artifactinfo) { + return [ + "repo": artifactinfo['Release repo'], + "revision": artifactinfo['Release repo revision'] + ] +} + +return this \ No newline at end of file diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..fa32ebf --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,408 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + +class ImportOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String m_filter // Only import files/paths/folders that match the filter + String m_excludes // Exclude files/paths/folders that match the exclude filter + boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_flatten // Flatten files and folders imported + boolean m_includemetainfilter // Make sure meta is also included in filter + boolean m_clean // Clean import folder before obtaining artifacts + boolean m_appendtoparentmeta // Append to parent meta + String m_sourceselect // Where are the artifacts coming from + + String m_jobname + def m_buildnumberorselector + + String m_repo + String m_revision + + boolean m_showdetails + + ImportOptions(script, + String sourceselect = 'Jenkins job artifact', + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false, + boolean flatten = false, + boolean includemetainfilter = true, + boolean clean = true, + appendtoparentmeta = true, + showdetails = false) { + m_script = script + m_sourceselect = sourceselect.trim() + m_incrementversionnumber = incrementversionnumber + m_filter = filter.trim() + m_excludes = excludes.trim() + m_fingerprint = fingerprint + m_nested = nested + m_flatten = flatten + m_includemetainfilter = includemetainfilter + m_clean = clean + m_appendtoparentmeta = appendtoparentmeta + m_showdetails = showdetails + } + + String show() { + return """m_incrementversionnumber: ${m_incrementversionnumber} + m_filter: ${m_filter} + m_excludes: ${m_excludes} + m_fingerprint: ${m_fingerprint} + m_nested: ${m_nested} + m_flatten: ${m_flatten} + m_includemetainfilter: ${m_includemetainfilter} + m_clean: ${m_clean} + m_appendtoparentmeta: ${m_appendtoparentmeta} + m_sourceselect: ${m_sourceselect} + m_jobname: ${m_jobname} + m_buildnumberorselector: ${m_buildnumberorselector} + m_repo: ${m_repo} + m_revision: ${m_revision} + """ + } + + void setsourceselect(String sourceselect) { + sourceselect = sourceselect.trim() + switch (sourceselect) { + case 'Jenkins job artefact': + m_sourceselect = 'Jenkins job artifact' + break + case 'Jenkins job artifact': + case 'Release repository': + m_sourceselect = sourceselect + break + default: + m_script.error("Unknown type of artifact source select: ${sourceselect}") + } + } + + void setincrementversionnumber(boolean incrementversionnumber = true) { + m_incrementversionnumber = incrementversionnumber + } + + void setfilter(String filter) { + m_filter = filter.trim() + } + + void setexcludes(String excludes) { + m_excludes = excludes.trim() + } + + void setfingerprint(boolean fingerprint = true) { + m_fingerprint = fingerprint + } + + void setnested(boolean nested = true) { + m_nested = nested + } + + void flattenimports(boolean flatten = true) { + m_flatten = flatten + } + + void includemetainfilter(boolean includemetainfilter = true) { + m_includemetainfilter = includemetainfilter + } + + void clean(boolean clean = true) { + m_clean = clean + } + + void setappendtoparentmeta(boolean appendtoparentmeta) { + m_appendtoparentmeta = appendtoparentmeta + } + + void jenkinsjob(String jobname, String buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: buildnumberorselector + ] + setsourceselect('Jenkins job artifact') + } + + void jenkinsjob(String jobname, int buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: "${buildnumberorselector}" + ] + setsourceselect('Jenkins job artifact') + } + + void releaserepo(String repo, String revision) { + m_repo = repo.trim() + m_revision = revision.trim() + setsourceselect('Release repository') + } + + void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) { + jenkinsjob(jobname, buildnumberorselector) + releaserepo(repo, revision) + setsourceselect(sourceselect) + } + + void showdetails(boolean showdetails) { + m_showdetails = showdetails + } +} + +ImportOptions getNewImportOptions() { + return new ImportOptions(this) +} + +def formFilter(ImportOptions importoptions) { + def String filter = importoptions.m_filter + if (filter != '') { + if (importoptions.m_includemetainfilter) { + filter = 'meta/**,' + filter + } + } + echo "Import options filter is: ${filter}" + return filter +} + +def conditionalClean(ImportOptions importoptions, String destpath) { + if (importoptions.m_clean) { + echo "Cleaning ${destpath}" + dir(destpath) { + deleteDir() + } + } +} + +def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Jenkins job artifact') { + error("Import options settings have not been set for: Jenkins job artifact") + return + } + + conditionalClean(importoptions, destpath) + + echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" + echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" + copyArtifacts( + filter: formFilter(importoptions), + excludes: importoptions.m_excludes, + fingerprintArtifacts: importoptions.m_fingerprint, + projectName: importoptions.m_jobname, + selector: importoptions.m_buildnumberorselector, + flatten: importoptions.m_flatten, + target: destpath + ) + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) + if (importoptions.m_incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def rewritemetafileasreleased(meta, String filepath) { + // Modify the meta info to say that is has already been released + meta.markreleased() + meta.saveJsonSerialised(filepath) +} + +def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + conditionalClean(importoptions, destpath) + + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + + echo "Getting associated metadata from path: ${destpath}" + def collectedmetalist = importedmeta.getassociated(destpath) + for(def collectedmeta in collectedmetalist) { + echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}" + } + + echo "Cloning associated repos inside path: ${destpath}" + for(def collectedmeta in collectedmetalist) { + def relatedmeta = collectedmeta.metainfo + def relatedoffsetpath = collectedmeta.accumulatedpath + def cloneurl = relatedmeta.releaseinfo.releaserepoURL + def clonerevision = "refs/tags/${relatedmeta.releasetag()}" + def clonepath = "${destpath}/${relatedoffsetpath}" + if (relatedmeta.componentidentifier() != '_' && + cloneurl != '') { + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } else { + echo "Not importing component that would be placed at path: ${clonepath}" + } + } + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(importedmeta, metafilepath) + + if (importoptions.m_incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + conditionalClean(importoptions, destpath) + + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision + echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(importedmeta, metafilepath) + + if (importoptions.m_incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimport(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_showdetails) { + echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" + } + switch (importoptions.m_sourceselect) { + case 'Jenkins job artifact': + case 'Jenkins job artefact': + return artifactimportfrombuildjob(meta, importoptions, destpath) + case 'Release repository': + if (importoptions.m_nested) { + return hierarchyartifactimportfromrepo(meta, importoptions, destpath) + } + return artifactimportfromrepo(meta, importoptions, destpath) + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + +// Deprecated (use above) +def artifactimport(meta, + String sourceselect, + String jenkinsjob, + buildnumberorselector, + String repo, + String revision, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false) { + echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" + def importoptions = importer.getNewImportOptions() + + importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + importoptions.setnested(nested) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + int buildnumber, + String destpath, + boolean incrementversionnumber = true, + String filter = '**', + String excludes = '', + boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" + def importoptions = importer.getNewImportOptions() + + importoptions.jenkinsjob(buildjob, " ${buildnumber}") + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + buildnumberorselector, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" + def importoptions = importer.getNewImportOptions() + + importoptions.jenkinsjob(buildjob, buildnumberorselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + +// Import the artifacts of a named child component found below parent meta's hierarchy +def artifactimport(meta, + parentmeta, + String parentmetapath, + String childproduct, + String childsubcomponent, + ImportOptions importoptions, + String destinationfolder) { + // Import from parent meta + def childmeta = meta.getNewMetaObject() + def childmeta2 + echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" + echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" + def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) + if (findresults.matchfound) { + childmeta = findresults.matchedmetaitem.metainfo + if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) { + importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) + } else { + importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) + } + childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) + } else { + error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") + } + return childmeta2 +} + +return this; diff --git a/install/installhelper.groovy b/install/installhelper.groovy new file mode 100644 index 0000000..8dbf23e --- /dev/null +++ b/install/installhelper.groovy @@ -0,0 +1,136 @@ +import hudson.FilePath + +// Transfer file from intermediate position to final position; +// creating necessary folders on the way; +// deleting intermediate file +def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { + def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() + + def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" + def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" + def export_filepath = new hudson.FilePath(channel, abs_export_out) + def export_out_folder = export_filepath.getParent() + def export_out_folder_string = export_out_folder.toString() + def export_out_file = export_filepath.getName() + + echo "Creating folder ${export_out_folder_string}" + export_out_folder.mkdirs() + + echo "Copying from ${pathToIntermediate} to ${abs_export_out}" + def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) + intermediate_filepath.copyTo(export_filepath) + intermediate_filepath.delete() +} + +// Given details about artifacts to import and details of installers to build, import artifacts that are required +def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { + artifactFetched = [:] + + if (showdetails) { + echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") + } + def importoptions = importer.getNewImportOptions() + importoptions.showdetails(showdetails) + for (installer in toBuild) { + def installerinfo = installerDetails.get(installer) + if (showdetails) { + echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") + } + def artifactKeys = installerinfo[1] + if (artifactKeys instanceof java.lang.String) { + artifactKeys = [ artifactKeys ] + } + if (showdetails) { + echo ("artifactKeys:\n${artifactKeys}") + } + for (artifactKey in artifactKeys) { + artifactDetail = artifactDetails.get(artifactKey) + if (showdetails) { + echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") + } + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail + if (artifactFetched.get(artifactKey) == null) { + importoptions.jenkinsjob(sourceProject, sourceselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(toCopy) + importoptions.setexcludes(notToCopy) + + def importmeta = importer.artifactimport(meta, importoptions, destFolder) + artifactFetched.put(artifactKey, importmeta) + } + } + } + if (showdetails) { + echo ("artifactFetched:\n${artifactFetched}") + } + return artifactFetched +} + +@NonCPS +def checkISMfilebackend(String xml) { + def String message = "" + + def parsedxml = new XmlParser(true, true, true).parseText(xml) + def pathvariableinfo = [:] + def exitcode = 0 + // Load up a map of path variable and path pairs + // Ignore blank path definitions + // Ignore the workspace definition which should have the only absolute path + parsedxml.each{ table -> + if (table.@name == "ISPathVariable") { + table['row'].each{ row -> + def pathname = row['td'][0].value()[0] + def path = row['td'][1].value()[0] + if (path && pathname != 'WORKSPACE') { + pathvariableinfo[pathname] = path + } + } + } + } + // Check that each path variable is defined beginning with a path variable or the WORKSPACE one + pathvariableinfo.each{ pair -> + message += "pair key: ${pair.key} value: ${pair.value}\n" + def path = pair.value + def pathvarmatcher = path =~ /<(.*)>.*/ + if (pathvarmatcher.matches()) { + def pathvarfound = pathvarmatcher.group(1) + if (pathvarfound != 'WORKSPACE') { + if (!pathvariableinfo.containsKey(pathvarfound)) { + message += "Undefined path variable ${pathvarfound} found\n" + exitcode = 1 + } + } + } else { + message += "Path: ${path} does not begin with a path variable\n" + exitcode = 1 + } + } + return [exitcode, message] +} + +def checkISMfile(String ISMfile) { + echo "Checking ISM file: ${ISMfile}" + def xml = readFile(ISMfile) + //echo "ISM file xml: ${xml}" + (exitcode, message) = checkISMfilebackend(xml) + echo message + if (exitcode > 0) { + error "ISM file errors found" + } else { + echo "Checked ISM file - ok" + } +} + +certificatethumbprints = [:] +certificatethumbprints['floreana'] = '0FDEB0A227982C877146CA595091173AC9655479' // Authoring takes place on this machine - so this is what should be in the ISM file already +certificatethumbprints['hawaii'] = '49FD1B361CFF35014B81940C5B046543045DD34B' + +def selectSigningCertificate(String nodeName, String ISMfile) { + def masterpattern = '0FDEB0A227982C877146CA595091173AC9655479' + echo "Substituting occurrences of ${masterpattern} with ${certificatethumbprints[nodeName]} in ISM file: ${ISMfile}" + + textutils_subst = load("${autobuilder.commonscripts()}/textutils/subst.groovy") + textutils_subst.substitute(ISMfile, masterpattern, certificatethumbprints[nodeName]) +} + +return this; diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..6715145 --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,1193 @@ +import java.nio.file.Paths +import java.nio.file.Path +import groovy.json.JsonSlurperClassic + +abstract class GISJsonSerialised implements Serializable { + def script + + GISJsonSerialised(script) { + this.script = script + } + + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + private String format + private String productname + private String subcomponentname + private String variant + private String uniqueid + + ProductInfo(script, String productname = '', String subcomponentname = '', String variant = '') { + super(script) + this.format = '0.1' + this.productname = productname + this.subcomponentname = subcomponentname + this.variant = variant + this.uniqueid = UUID.randomUUID().toString() + } + + void set(String productname, String subcomponentname, String variant = '') { + // Make sure the fields can be used in git tags + def int pstatus = 0 + if (script.isUnix()) { + pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) + } + if (pstatus > 0) { + script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") + } + def int sstatus = 0 + if (script.isUnix()) { + sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) + } + if (sstatus > 0) { + script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") + } + if (variant) { + def int vstatus = 0 + if (script.isUnix()) { + vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) + } + if (vstatus > 0) { + script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") + } + } + this.productname = productname + this.subcomponentname = subcomponentname + this.variant = variant + } + + void set(ProductInfo otherinfo) { + this.productname = otherinfo.productname + this.subcomponentname = otherinfo.subcomponentname + this.variant = otherinfo.variant + this.uniqueid = otherinfo.uniqueid + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + variant = rawmeta['variant'] ?: '' + uniqueid = rawmeta['uniqueid'] ?: uniqueid + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'variant' : variant, + 'uniqueid': uniqueid, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String format + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = '', sourcerevision = '', gitrevision = '', remoteurl = '', remoteinfo = '') { + super(script) + this.format = '0.1' + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + GitInfo(GitInfo another) { + super(another.script) + this.sourcepath = another.sourcepath + this.sourcerevision = another.sourcerevision + this.gitrevision = another.gitrevision + this.remoteurl = another.remoteurl + this.remoteinfo = another.remoteinfo + } + + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + void set(GitInfo otherinfo) { + this.sourcepath = otherinfo.sourcepath + this.sourcerevision = otherinfo.sourcerevision + this.gitrevision = otherinfo.gitrevision + this.remoteurl = otherinfo.remoteurl + this.remoteinfo = otherinfo.remoteinfo + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildParameter extends GISJsonSerialised { + private String name + private String valuetype + private String value + + JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { + super(script) + this.name = name + this.valuetype = valuetype + this.value = value + } + + void set(String name, String valuetype, String value) { + this.name = name + this.valuetype = valuetype + this.value = value + } + + void deserialiseJson(rawmeta) { + name = rawmeta['name'] + valuetype = rawmeta['valuetype'] + value = rawmeta['value'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'name' : name, + 'valuetype' : valuetype, + 'value' : value, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String format + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + private ArrayList parameters + private boolean isfullbuild + + JenkinsBuildInfo(script, String buildjobname = '', int buildnumber = 0, String buildmachine = '', String builddate = '', boolean isfullbuild = true) { + super(script) + this.format = '0.1' + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + this.isfullbuild = isfullbuild + parameters = new ArrayList() + } + + void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + this.isfullbuild = isfullbuild + } + + void set(boolean isfullbuild) { + this.isfullbuild = isfullbuild + } + + boolean isFullBuild() { + return isfullbuild + } + + void addParameter(String name, value) { + def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) + parameters.add(p) + return p + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] + parameters = new ArrayList() + if (rawmeta['isfullbuild'] != null) { + for (rawparameter in rawmeta['parameters']) { + def p = new JenkinsBuildParameter(script) + p.deserialiseJson(rawparameter) + parameters.add(p) + } + } + } + + List parametersserialised() { + def associated = [] + for (p in parameters) { + associated.add(p.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + 'isfullbuild' : isfullbuild, + 'parameters' : parametersserialised(), + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + private String format + private String buildstampprefix + private int revision + private String buildstamp + + BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { + super(script) + this.format = '0.1' + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + } + + BuildStampInfo(BuildStampInfo another) { + super(another.script) + this.buildstampprefix = another.buildstampprefix + this.revision = another.revision + this.buildstamp = another.buildstamp + } + + void set(String buildstampprefix, int revision, String buildstamp) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + } + + void set(BuildStampInfo other) { + set(other.buildstampprefix, other.revision, other.buildstamp) + } + + // Override the buildstamp + void override(String buildstamp) { + this.buildstamp = buildstamp + script.echo("Build stamp overridden to: ${this.buildstamp}") + def s = this.buildstamp.split('\\.') + if (s.size() != 4) { + script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") + } + this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" + script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") + this.revision = s[3].toInteger() + script.echo("Revision overridden to: ${this.revision}") + } + + String buildstampcommasep() { + def String buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + + void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") + revision = newrevision + def List buildstampsplit = buildstamp.split('\\.') + buildstampsplit[-1] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + script.echo("Build stamp now ${buildstamp}") + } + + void set(String buildstampprefix, int revision) { + buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' + this.buildstampprefix = buildstampprefix + this.buildstamp = buildstampprefix + "0" + changerevision(revision) + } + + void set(String buildstampprefix, String revision = '0') { + set(buildstampprefix, revision.toInteger()) + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + private String format + private String releaserepoURL + private ArrayList releaseitems + private boolean released + + ReleaseInfo(script, String releaserepoURL = '', boolean released = false) { + super(script) + this.format = '0.1' + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() + this.released = released + } + + void set(String releaserepoURL = '') { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + script.echo("Adding release item: ${releaseitem} to release items") + releaseitems.add(releaseitem) + } + + void markreleased() { + this.released = true + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + released = rawmeta['released'] ?: false + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), + 'released' : released + ] + return data + } +} + +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = '', configurationURL = '', configurationversion = '', configurationSHA = '') { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath = '', String sum = '') { + super(script) + this.filepath = filepath + this.sum = sum + } + + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String format + private String artifactpath + private ArrayList checksumpairs + + ChecksumSet(script, artifactpath = '') { + super(script) + this.format = '0.1' + this.artifactpath = artifactpath + checksumpairs = new ArrayList() + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } +} + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + checksumsets = new ArrayList() + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + +class PublishableInfo extends GISJsonSerialised { + private String format + private String filepath + + PublishableInfo(script, String filepath = '') { + super(script) + this.format = '0.1' + this.filepath = filepath + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + filepath = rawmeta['filepath'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'filepath' : filepath, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private static String METAFOLDERNAME = 'meta' + private static String METAFILENAME = 'info.json' + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String format = '0.2' + + abstract class RelatedMetaInfo extends GISJsonSerialised { + String format + + RelatedMetaInfo(script, format) { + super(script) + this.format = format + } + + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + abstract String getrelatedid() + abstract void markreleased() + } + + class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { + private MetaInfo meta + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_1(script, String offsetpath = '', MetaInfo meta = null, String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.1') + this.offsetpath = offsetpath + this.uniqueid = uniqueid + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] ?: uniqueid + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid, + 'meta' : meta.serialisationdata(), + ] + return data + } + + String getrelatedid() { + return uniqueid + } + + void markreleased() { + meta.releaseinfo.markreleased() + } + } + + class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { + private String importpath + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_2(script, String importpath = '', String offsetpath = '', String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.2') + this.importpath = importpath + this.offsetpath = offsetpath + this.uniqueid = uniqueid + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid + ] + return data + } + + String getrelatedid() { + return uniqueid + } + + void markreleased() { + // Do nothing + } + } + + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + private ChecksumInfo checksuminfo + private ArrayList configurationinfo + private ArrayList publishableinfo + private LinkedHashMap custominfo + private ArrayList associatedmeta + + MetaInfo(script) { + super(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) + configurationinfo = new ArrayList() + publishableinfo = new ArrayList() + custominfo = new LinkedHashMap() + associatedmeta = new ArrayList() + } + + void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") + releaseinfo.markreleased() + for (associated in associatedmeta) { + associated.markreleased() + } + } + + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return configuration + } + + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") + publishableinfo.add(pi) + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") + configurationinfo.add(ci) + return ci + } + + void addCustomInfo(String customId, String customValue) { + custominfo.put(customId.trim(), customValue) + } + + String getCustomInfo(String customId) { + return custominfo.getAt(customId.trim()) + } + + def isFullBuild() { + return jenkinsbuildinfo.isFullBuild() + } + + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(metaitem.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { + script.error("Unsupported meta format found") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + custominfo = rawmeta['custominfo'] + for (rawrelatedmeta in rawmeta['associatedmeta']) { + if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { + def rm = new RelatedMetaInfo_v0_1(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } else if (rawrelatedmeta['format'] == '0.2') { + def rm = new RelatedMetaInfo_v0_2(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + } + + def deserialiseJson(String filepath) { + script.echo("Deserialising JSON at file: ${filepath}") + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + def deserialiseJsonSlurper(String filepath) { + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) + deserialiseJson(rawmeta) + } + + String relativize(String mypath) { + String pathRelative + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + } else { + pathRelative = mypath + script.echo("Path relative is unchanged: ${pathRelative}") + } + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') + // Strip leading / if any + pathRelative = pathRelative.replaceAll('^/+', '') + + script.echo("Path relative is ${pathRelative}") + + return pathRelative + } + + static String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + + MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { + MetaInfo mi = new MetaInfo(script) + + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + metafilepath() + if (script.fileExists(importedmetafile)) { + script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") + mi.deserialiseJson(importedmetafile) + + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") + addConfigurationInfo('', givenRepo, givenRevision, givenSHA) + } + return mi + } + + def copyrelatedmeta() { + script.echo("Copying related input meta data to output meta") + for (relatedmetainfo in associatedmeta) { + def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" + script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") + script.fileOperations([ + script.folderCreateOperation(folderPath: destpath), + script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) + ]) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), + 'custominfo' : custominfo, + 'associatedmeta' : associatedmetaserialised(), + ] + return data + } + + static String folder() { + return METAFOLDERNAME + } + + def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = METAARTIFACTS + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } + saveJsonSerialised(metafilepath) + return artifactlist + } + + String componentidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + } + + String componentversionuniqueidentifier() { + return componentidentifier() + "_" + buildstampinfo.buildstamp + } + + String globalreleasetag() { + return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber + } + + String releasetag() { + return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber + } + + String releasetagfordebpackage() { + return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber + } + + class CollectedMeta { + private MetaInfo metainfo + private String accumulatedpath + private String accumulatedmetapath + + CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { + this.metainfo = metainfo + this.accumulatedpath = accumulatedpath + this.accumulatedmetapath = accumulatedmetapath + } + } + + class IterationInfo { + private ArrayList metacollection + private Map metamap + private String comment + private int level + private String gatheredpath + private String gatheredmetapath + private boolean limiteddepth + private int maxdepth + + IterationInfo(ArrayList metacollection, + Map metamap, + String comment = '', + int level = 0, + String gatheredpath = '', + String gatheredmetapath = '', + boolean limiteddepth = false, + int maxdepth = 0) { + this.metacollection = metacollection + this.metamap = metamap + this.comment = comment + this.level = level + this.gatheredpath = gatheredpath + this.gatheredmetapath = gatheredmetapath + this.limiteddepth = limiteddepth + this.maxdepth = maxdepth + } + + String show() { + return "Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}" + } + + void add(CollectedMeta item) { + metacollection.add(item) + metamap.put(item.metainfo.productinfo.uniqueid, item) + } + } + + def openchildmeta(IterationInfo ii, RelatedMetaInfo relatedmetainfo) { + def MetaInfo mi = new MetaInfo(script) + + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" + def boolean imported + // If this has already been read in from a file + if (relatedmetainfo.getrelatedid() in ii.metamap) { + //script.echo("Already read in meta associated with uniqueid: ${relatedmetainfo.getrelatedid()} - Using it") + mi = ii.metamap[relatedmetainfo.getrelatedid()].metainfo + imported = true + } else if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + imported = true + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") + imported = false + } + + def childmetainfo = [ + 'imported': imported, + 'nextgatheredpath': nextgatheredpath, + 'nextgatheredmetapath': nextgatheredmetapath, + 'childmeta': mi + ] + return childmetainfo + } + + def walkrelated_v0_1(IterationInfo ii) { + //script.echo("Walk related v0.1: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } else { + script.error("There is non v0.1 format related meta info under v0.1 related meta info") + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + def walkrelated(IterationInfo ii) { + //script.echo("Walk related: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + //script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") + if (relatedmetainfo.format == '0.2') { + def childmetainfo = openchildmeta(ii, relatedmetainfo) + if (childmetainfo.imported) { + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + childmetainfo.nextgatheredpath, + childmetainfo.nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = childmetainfo.childmeta.walkrelated(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + // Walks all layers of associated data only + def allrelatedmeta(String metapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + return walkrelated(new IterationInfo(metacollection, metamap, '', 0, '', metapath)) + } + + ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) + script.echo("Release walker iteration info at start:\n${ii.show()}") + def results = walkrelated(ii) + return results.metacollection + } + + // Walks 1 layer of associated data only + def getassociated(String rootpath) { + return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself) + } + + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + def existingentrypathlength = existingentry.accumulatedpath.split('/').size() + def newentrypathlength = metaitem.accumulatedpath.split('/').size() + if (existingentrypathlength > newentrypathlength) { + releasemetacollection.put(uniquestring, metaitem) + } + } else { + releasemetacollection.put(uniquestring, metaitem) + } + return releasemetacollection + } + + Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") + + def Map versionuniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() + versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) + } + + return versionuniquecollection + } + + Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") + + def Map uniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentidentifier() + uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) + } + + return uniquecollection + } + + MetaInfo getNewMetaObject(script) { + return new MetaInfo(script) + } + + def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { + def results = [ + 'matchedmetaitem' : new CollectedMeta(this), + 'matchfound' : false + ] + def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") + for (dependentmetaitem in walkresults.metacollection) { + script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") + if (dependentmetaitem.metainfo.productinfo.productname == productname && + dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { + script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") + results.matchedmetaitem = dependentmetaitem + results.matchfound = true + break + } + } + return results + } +} + +MetaInfo metareset(script) { + def mi = new MetaInfo(script) + + echo "Resetting meta folder" + dir(mi.folder()) { + deleteDir() + } + + return mi +} + +MetaInfo metafromfolder(String folder = '') { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + +MetaInfo metafromfolderSlurper(String folder = '') { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + +return this; diff --git a/packaging/debian.groovy b/packaging/debian.groovy new file mode 100644 index 0000000..9f2f4d3 --- /dev/null +++ b/packaging/debian.groovy @@ -0,0 +1,111 @@ +class DebRootDefinition { + String ROOTPATH + String PACKAGE_PREFIX + String PACKAGE_FOLDER_NAME + String PACKAGE_NAME + String UNIQUE_PACKAGE_NAME + + String PACKAGE_SOURCE_ROOT + String PACKAGE_SOURCE_FOLDER_NAME + String DEBPATH + + String ETC + String GIS_OPT_BIN + String GIS_OPT_LIB + String GIS_OPT_SCRIPTS_APP + String GIS_OPT_SCRIPTS_EL + String GIS_OPT_SCRIPTS_CONTROL + String GIS_OPT_WWW + String GIS_OPT_OTHER + String GIS_OPT_KERNEL + String GIS_OPT_LOGIC + String GIS_OPT_PRINTHEAD + String GIS_OPT_BOOT + + String USR_BIN + String USR_LIB_ARCH + String LIB_FIRMWARE + String HTTP_DEV_DIV + String LIB_SYSTEMD + + DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix) { + ROOTPATH = rootpath + PACKAGE_PREFIX = packageprefix + PACKAGE_FOLDER_NAME = packageprefix + PACKAGE_NAME = packageprefix + "_armhf.deb" + UNIQUE_PACKAGE_NAME = packageprefix + "_" + buildstamp + "_armhf.deb" + + PACKAGE_SOURCE_ROOT = packagesourceroot + PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" + DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" + + ETC = "${DEBPATH}/etc" + inpackagepathsuffix + GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + inpackagepathsuffix + GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + inpackagepathsuffix + GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + inpackagepathsuffix + GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + inpackagepathsuffix + GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" + inpackagepathsuffix + GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + inpackagepathsuffix + GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + inpackagepathsuffix + GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" + inpackagepathsuffix + GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + inpackagepathsuffix + GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + inpackagepathsuffix + GIS_OPT_BOOT = "${DEBPATH}/opt/gis/boot" + inpackagepathsuffix + + USR_BIN = "${DEBPATH}/usr/bin" + inpackagepathsuffix + USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + inpackagepathsuffix + LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + inpackagepathsuffix + HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + inpackagepathsuffix + LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + inpackagepathsuffix + } +} + +def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix = '') { + return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) +} + +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { + withCredentials([usernamePassword(credentialsId: credentials_id, + usernameVariable: 'SUDO_USERNAME', + passwordVariable: 'SUDO_PASSWORD')]) { + def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" + def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" + try { + sh """#!/bin/bash + # Exit if any command fails + set -xe + + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + + ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" + + mkdir -v -p "${packageout}" + echo "Building package:" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + + mkdir -v -p "${uniquepackageout}" + cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" + + echo "Showing package:" + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package contents:" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + + echo "Showing package lintian output:" + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true + + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + """ + } + catch (Exception e) { + echo "Exception occurred" + sh """#!/bin/bash -v + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + + exit 1 + """ + } + } +} + +return this; \ No newline at end of file diff --git a/platform/lineendings.groovy b/platform/lineendings.groovy new file mode 100644 index 0000000..9657b0d --- /dev/null +++ b/platform/lineendings.groovy @@ -0,0 +1,5 @@ + +// Put ${nl} into strings when wanting a newline +nl = System.properties['line.separator'] + +return this; diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..ea090f0 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,174 @@ +import groovy.io.FileType + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(filteredreleaserepofolder) { + deleteDir() + } +} + +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { + return "Software/${productline} ${softwarerelease}/${productuniquefolder(releasemeta)}" +} + +def uniqueabsoluterreleasefolder(String productline, String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(productline, softwarerelease, releasemeta)}" +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def guessreporootsuffix(componentURL) { + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + return guess +} + +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.metainfo + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" + def publishableitems = releasemeta.publishableinfo + + def ArrayList publishablepaths = new ArrayList() + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + } + } + if (publishablepaths.size() > 0) { + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone + } + else { + summaryappend("None ") + } +} + +def publish(hostedftpsiteinfo, String customer, String finalreleaserepofolder = releaserepofolder) { + summaryappend(" FTP operations: ") + + def String ftproot = "${hostedftpsiteinfo.workingdirectory}${customer}" + dir(finalreleaserepofolder) { + summaryappend("FTP operations: Transferring to ${ftproot}") + sh """#!/bin/bash + ncftpput -u ${hostedftpsiteinfo.username} -p ${hostedftpsiteinfo.password} -R ${hostedftpsiteinfo.url} "${ftproot}" . + """ + summaryappend(", Transferred ") + } +} + +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdirs(); + } + dirFrom.eachFile(FileType.FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { + def releasemeta = releaseinfo.metainfo + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) + + summaryappend("\n") +} + +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + def productline = releasemeta.productinfo.productname + // Walk the meta information to a max depth, looking for artifacts to publish + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) + + echo "Version unique components:" + def String versionuniquecomponent = "" + for (key in componentReleaseInfoVersionUnique.keySet()) { + versionuniquecomponent += "${key}\n" + } + echo versionuniquecomponent + + def itempathstopublish = convertitemstopublish(itemstopublishstring) + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + recallcomponent(productline, releaseinfo, softwarerelease, itempathstopublish) + } + gh.stripgithistory(releaserepofolder) + publish(hostedftpsiteinfo, customer) + + return [summarystring, markunstable, unstablereason] +} + +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(hostedftpsiteinfo, releasemeta, releasemetapath, '', customer, softwarerelease, maxdepth) +} + +return this; diff --git a/publish/package.groovy b/publish/package.groovy new file mode 100644 index 0000000..68c41f3 --- /dev/null +++ b/publish/package.groovy @@ -0,0 +1,181 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHONBRANCH = "origin/3.6.3_linux" + } + else { + PYTHONBRANCH = "origin/3.6.3_windows" + } + PYTHONDIR = "${WORKSPACE}/python" + PYTHONROOTDIR = 'python' + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + } + stage('Clean exports') { + steps { + script { + PACKAGEDEST = "${WORKSPACE}/package" + cleanfolders = [ + PACKAGEDEST, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying package artifacts') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + def packageimportoptions = importer.getNewImportOptions() + packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) + packageimportoptions.setincrementversionnumber(true) + packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) + + PACKAGE_FOLDER = "${PACKAGEDEST}/package" + UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" + } + } + } + stage('Aptly environment') { + steps { + script { + echo "Aptly config file is ${APTLY_CONFIG_FILE}" + sh """#!/bin/bash -v + aptly config show + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Create repo - if not already there') { + steps { + script { + echo "Repo name is ${REPO_NAME}" + sh """#!/bin/bash -v + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then + echo "aptly repo ${REPO_NAME} does not exist; Creating it" + aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + fi + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Drop published repo - if any') { + steps { + script { + sh """#!/bin/bash -v + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + + stage('Add/update package in repo') { + steps { + script { + echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" + sh """#!/bin/bash -v + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Create snapshot') { + steps { + script { + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" + // Convert prefix to regular expression + PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") + PACKAGE_VERSION_REGEX = '' + i = 0 + while(true) { + if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + break; + } + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + PACKAGE_VERSION_REGEX += '\\.' + i++ + } + sh """#!/bin/bash -v + # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' + if [ \$? -ne 0 ]; then exit 1; fi + aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Publish snapshot') { + steps { + script { + sh """#!/bin/bash -v + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy new file mode 100644 index 0000000..ce6a805 --- /dev/null +++ b/publish/promptingpublish.groovy @@ -0,0 +1,182 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "origin/master" + SOURCEDIR = "${WORKSPACE}/whole" + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + } + } + } + stage('Identify customer to publish to') { + steps { + script { + hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") + def List customers = hostedftb.getCustomerList() + timeout(time: 10, unit: 'MINUTES') { + customer = input( + message: 'Please select customer to publish to', + ok: 'Ok', + parameters: [ + choice(name: 'customer', choices: customers, description: 'Which customer?') + ] + ) + echo "Chosen customer is: ${customer}" + } + } + } + } + stage('Identify software release to publish') { + steps { + script { + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + def allreleases = gh.gettaglist(SOURCEDIR) + def releases = new ArrayList() + if (FILTER_REDUNDANT_RELEASES == 'false') { + echo("Showing all releases") + releases = allreleases + } else { + echo("Showing filtered releases") + for (release in allreleases) { + if (!release.toLowerCase().startsWith("superceded") && + !release.toLowerCase().startsWith("superseded") && + !release.toLowerCase().startsWith("erroneous") && + !release.toLowerCase().startsWith("faulty") && + !release.toLowerCase().startsWith("ignore") && + !release.toLowerCase().startsWith("inmainline")) { + echo("Showing release: ${release}") + releases.add(release) + } else { + echo("Not showing release: ${release}") + } + } + } + timeout(time: 10, unit: 'MINUTES') { + release = input( + message: 'Please select the software version', + ok: 'Ok', + parameters: [ + choice(name: 'release', choices: releases, description: 'Which software release?') + ] + ) + echo "Chosen software version is: ${release}" + } + } + } + } + stage('Checkout release repo at selected software version') { + steps { + script { + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${release}" + SOURCEDIR = "${WORKSPACE}/whole" + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + } + } + } + stage('Get release artifact info') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Select which items from software release to publish') { + steps { + script { + publishitems = load("${autobuilder.commonscripts()}/publish/publishitems.groovy") + timeout(time: 10, unit: 'MINUTES') { + def itemsobject = publishitems.createItemSelector( + 'Select software items', + 'Software items', + releasemeta, + SOURCEDIR, + ITEMSDEFAULTLIST, + MAX_META_DEPTH, + '20', + 'PT_MULTI_SELECT') + selecteditems = input( + message: 'Select which items from the release are to be published', + ok: 'Ok', + parameters: [ + itemsobject.multiSelect() + ] + ) + echo "Chosen items are: ${selecteditems}" + } + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, SOURCEDIR, selecteditems, customer, release, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..ad33df9 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,92 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + def hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") + def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/publishitems.groovy b/publish/publishitems.groovy new file mode 100644 index 0000000..9f6c368 --- /dev/null +++ b/publish/publishitems.groovy @@ -0,0 +1,129 @@ +import com.cwctravel.hudson.plugins.extended_choice_parameter.ExtendedChoiceParameterDefinition + +class PublishableItemsSelection { + def script + def ArrayList preselectedmenuitems + def ArrayList menuitemdescriptions + def ArrayList menuitemsfull + def ExtendedChoiceParameterDefinition multiselect + + PublishableItemsSelection(script) { + this.script = script + preselectedmenuitems = new ArrayList() + menuitemdescriptions = new ArrayList() + menuitemsfull = new ArrayList() + } + + def init( + String name, + String description, + releasemeta, + String releasemetapath, + String preselectedext, + String metalevelsstringext, + String visibleItemCountExt, + String controltype) { + outputSelectionfromPreselect(preselectedmenuitems, preselectedext) + def Integer metalevelsext = metalevelsstringext.toInteger() + def Integer visibleItemCount = visibleItemCountExt.toInteger() + def metaitems = releasemeta.releasewalkerunique(metalevelsext, '', releasemetapath) + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + outputDescription(menuitemdescriptions, releaseinfo) + outputSelection(menuitemsfull, releaseinfo) + } + + script.echo("name is: ${name}") + script.echo("description is: ${description}") + script.echo("menuitemsfull is: ${menuitemsfull}") + script.echo("preselectedmenuitems is: ${preselectedmenuitems}") + script.echo("menuitemdescriptions is: ${menuitemdescriptions}") + script.echo("visibleItemCount is: ${visibleItemCount}") + script.echo("controltype is: ${controltype}") + + def seperator = "," + visibleItemCount = Math.min(visibleItemCount, menuitemsfull.size()) + script.echo("visibleItemCount is now: ${visibleItemCount}") + + multiselect = new ExtendedChoiceParameterDefinition( + name, // String name + controltype, // String type + menuitemsfull.join(seperator), // String value + "project name", // String projectName + "", // String propertyFile + "", // String groovyScript + "", // String groovyScriptFile + "", // String bindings + "", // String groovyClasspath + "", // String propertyKey + preselectedmenuitems.join(seperator),// String defaultValue + "", // String defaultPropertyFile + "", // String defaultGroovyScript + "", // String defaultGroovyScriptFile + "", // String defaultBindings + "", // String defaultGroovyClasspath + "", // String defaultPropertyKey + menuitemdescriptions.join(seperator),// String descriptionPropertyValue + "", // String descriptionPropertyFile + "", // String descriptionGroovyScript + "", // String descriptionGroovyScriptFile + "", // String descriptionBindings + "", // String descriptionGroovyClasspath + "", // String descriptionPropertyKey + "", // String javascriptFile + "", // String javascript + false, // boolean saveJSONParameterToFile + false, // boolean quoteValue + visibleItemCount, // int visibleItemCount + description, // String description + seperator) // String multiSelectDelimiter + } + + def outputSelectionfromPreselect(menuitems, preselectedext) { + script.echo ("preselectext: ${preselectedext}") + def menuitemstemp = preselectedext.split(',') + script.echo ("Menuitemstemp: ${menuitemstemp}") + + for (menuitem in menuitemstemp) { + menuitems.add(menuitem.trim()) + } + } + + def outputDescription(menuitems, releaseinfo) { + def meta = releaseinfo.metainfo + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } + } + + def outputSelection(menuitems, releaseinfo) { + def meta = releaseinfo.metainfo + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } + } + + def multiSelect() { + return multiselect + } +} + +PublishableItemsSelection createItemSelector( + String name, + String description, + releasemeta, + String releasemetapath, + String preselectedext, + String metalevelsstringext, + String visibleItemCountExt = '20', + String type = 'PT_CHECKBOX') { // Could also be "PT_MULTI_SELECT" + PublishableItemsSelection pis = new PublishableItemsSelection(this) + + pis.init(name, description, releasemeta, releasemetapath, preselectedext, metalevelsstringext, visibleItemCountExt, type) + + return pis +} + +return this; diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..2daf6f6 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,27 @@ + + +// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout +def allowed(String filepath) { + if (filepath.endsWith('/**') || filepath.endsWith('/')) { + def files = findFiles glob: filepath + if (files.length > 0) { + return + } else { + error "There are no files/folders under the publishable item: ${filepath}" + } + } + error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" +} + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(meta, publishable) { + filelist = [] + for (filepath in publishable) { + if (filepath != meta.METAARTIFACTS) { + allowed(filepath) + } + meta.addPublishable(filepath) + } +} + +return this; diff --git a/release/do.groovy b/release/do.groovy new file mode 100644 index 0000000..4dcb091 --- /dev/null +++ b/release/do.groovy @@ -0,0 +1,458 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +repoandtagsfile = "${summaryfolder}/repoandtags.txt" +repoandtagsstring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") + repoandtagsstring = "" + writeFile(file: repoandtagsfile, text: "Repository summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def repoandtagsappend(String summary) { + def String repoandtagscontents = readFile(file: repoandtagsfile) + writeFile(file: repoandtagsfile, text: repoandtagscontents + summary) + repoandtagsstring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + repoandtagsappend(" Release repo: ${releaserepo}") + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files - Note that this does not remove .git* + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo) { + repoandtagsappend(" Source repo: ${sourcerepo}") + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } else { + repoandtagsappend(" Source repo: None") + } +} + +def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + def boolean result = true + for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { + def filterpath = artifactfilter + if (buildinfo.accumulatedpath) { + filterpath = buildinfo.accumulatedpath + '/' + artifactfilter + } + // Copy relevant artifacts to temp folder + echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + try { + copyArtifacts( + filter: filterpath, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + ], + target: tempfolder + ) + } catch (Exception e) { + echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" + result = false + summaryappend("Does not exist:") + } + if (result) { + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath + dir(originalrootdir) { + sh("mv -v * ${repofolder}") + } + } + summaryappend("${artifactfilter},") + } + return result +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + + return changed +} + +def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem) + } + } + return taglist +} + +def String releasegitgetshacode(String tag) { + def existingshacode = "" + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + existingshacode = readFile(file: gitrevparseoutputfile).trim() + } + return existingshacode +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def String existingshacode = releasegitgetshacode(tag) + if (existingshacode) { // Tag exists + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with the one expected + // This is a problem - The tag is already at another position in git repo + setunstable("The release tag ${tag} has been used at another commit other than one expected - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${postcommitshacode})") + summaryappend("(${tag} - Could not add!)") + } else { + summaryappend("(${tag} - Already added)") + } + } else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + echo "Tag added: ${tag} with sha: ${postcommitshacode}" + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${shacode})") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + sh("git remote -v") + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + sh("git remote -v") + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def boolean isalreadyreleased(buildmeta, tag) { + def boolean alreadyreleased = buildmeta.releaseinfo.released + if (alreadyreleased) { + echo "Component is already marked as released: ${alreadyreleased}" + } else { + String commitshacode = releasegitgetshacode(tag) + if (commitshacode) { + echo "Tag: ${tag} already exists in this repo therefore marking as already released" + alreadyreleased = true + } else { + alreadyreleased = false + } + } + return alreadyreleased +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { + summaryappend(" Git operations: ") + def String buildstamptag = buildmeta.releasetag() + + dir(releaserepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) + + def String postcommitshacode + def boolean changed = false + if (alreadyreleased) { + summaryappend("Already-released,") + postcommitshacode = releasegitgetshacode(buildstamptag) + echo "Previous release sha code: ${postcommitshacode}" + } else { + releasegitadd() + + changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamptag) + } else { + releasegitreset() + summaryappend("No-commit,") + } + + postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + } + + ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } + dir(sourcerepo) { + def String srcbuildstamptag = buildmeta.globalreleasetag() + def String buildsha = buildmeta.gitinfo.gitrevision + + if (buildsha) { + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } else { + summaryappend(",No Src ") + } + } +} + +def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { + def buildmeta = buildinfo.metainfo + def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) + def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) + + checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) + + if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + } + + summaryappend("\n") + repoandtagsappend("\n") +} + +def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { + def int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + versioncomponent += "${key}\n" + } + echo versioncomponent + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) + def String versionuniquecomponent = '' + def String versionwithpathscomponent = '' + for (key in componentReleaseInfoVersionUnique.keySet()) { + versionuniquecomponent += "${key}\n" + versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" + } + echo 'Version unique components:' + echo versionuniquecomponent + + echo 'Version unique components with paths:' + echo versionwithpathscomponent + + if (DO_INTEGRITY_CHECK == "true") { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" + } + } + + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { + error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" + } + } + + def toplevelid = buildmeta.componentidentifier() + // For each component + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + echo "Releasing component: ${buildinfo.metainfo.componentidentifier()}" + if (buildinfo.metainfo.componentidentifier() != '___' && buildinfo.metainfo.componentidentifier() != '_') { + if (toplevelid != key) { + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + } + } + } + // Do the top level last + def toplevelbuildinfo = componentReleaseInfo[toplevelid] + releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..f565706 --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo) { + if (releaserepo?.trim()) { + vc.checkRepoExists(releaserepo) + } + meta.releaseinfo.set(releaserepo) +} + +return this; diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..82af511 --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + BUILD_ARTIFACTS = 'build' + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) + importoptions.setincrementversionnumber(false) + importoptions.setfilter('meta/**') + importoptions.setappendtoparentmeta(false) + buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + // Place the top level build stamp on all + OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/remoteaccess/ftp.groovy b/remoteaccess/ftp.groovy new file mode 100644 index 0000000..25b9143 --- /dev/null +++ b/remoteaccess/ftp.groovy @@ -0,0 +1,32 @@ +import org.apache.commons.net.ftp.FTPClient + +class FTPSiteInfo { + String url + String username + String password + String workingdirectory + + FTPSiteInfo(String url, String username, String password, String workingdirectory) { + this.url = url + this.username = username + this.password = password + this.workingdirectory = workingdirectory + } +} + +FTPSiteInfo createFTPSiteInfo(String url, String username, String password, String workingdirectory) { + FTPSiteInfo ftpsiteinfo = new FTPSiteInfo(url, username, password, workingdirectory) + return ftpsiteinfo +} + +FTPClient createFTPClient(FTPSiteInfo ftpsiteinfo) { + FTPClient ftpclient = new FTPClient() + ftpclient.connect(ftpsiteinfo.url) + ftpclient.enterLocalPassiveMode() + ftpclient.login(ftpsiteinfo.username, ftpsiteinfo.password) + ftpclient.changeWorkingDirectory(ftpsiteinfo.workingdirectory) + + return ftpclient +} + +return this; diff --git a/remoteaccess/hostedftp.groovy b/remoteaccess/hostedftp.groovy new file mode 100644 index 0000000..cf62d32 --- /dev/null +++ b/remoteaccess/hostedftp.groovy @@ -0,0 +1,28 @@ +ftpremote = load("${autobuilder.commonscripts()}/remoteaccess/ftp.groovy") + +def getHostedFTPSiteInfo() { + hostedftpsiteinfo = + ftpremote.createFTPSiteInfo("ftp.hostedftp.com", + "ftp@globalinkjetsystems.com", + "ramshorn42", + "/Customers/") + return hostedftpsiteinfo +} + +def getCustomerList() { + def List customerfilenames = new ArrayList() + def List customers = new ArrayList() + + def ftpsiteinfo = getHostedFTPSiteInfo() + def ftpclient = ftpremote.createFTPClient(ftpsiteinfo) + + customerfilenames = ftpclient.listNames() + for(customer in customerfilenames) { + def String converted = new String(customer.getBytes("ISO-8859-1"), "UTF-8") + customers.add(converted) + } + ftpclient.disconnect() + return customers +} + +return this; \ No newline at end of file diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy new file mode 100644 index 0000000..ae91cb0 --- /dev/null +++ b/staticanalysis/ccm.groovy @@ -0,0 +1,85 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + CCMURL = 'git@hydra:tools/ccm.git' + CCMBRANCH = 'origin/master' + CCMDIR = "${WORKSPACE}/ccm" + configurationhelper.gitCheckoutAndRegister(meta, 'ccm', CCMURL, CCMBRANCH, CCMDIR) + CCM = "${CCMDIR}/bin/CCM.exe" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEDIR = "${WORKSPACE}/source" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) + } + } + } + stage("Clean exports") { + steps { + script { + RESULTS = 'results' + cleanfolders = [ + RESULTS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Static analysis') { + steps { + script { + def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" + writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) + echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" + + dir(RESULTS) { + echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" + if (isUnix()) { + sh """#!/bin/bash + ${CCM}" "${ccmconfigurationfile}" > ccm.xml + """ + } else { + echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" + bat """ + ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml + """ + } + } + recordIssues(tools: [ + ccm(pattern: "**/ccm.xml", reportEncoding: 'UTF-8'), + ]) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/staticanalysis/cpd.groovy b/staticanalysis/cpd.groovy new file mode 100644 index 0000000..c04649c --- /dev/null +++ b/staticanalysis/cpd.groovy @@ -0,0 +1,152 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHONBRANCH = 'origin/3.6.3_linux' + } else { + PYTHONBRANCH = 'origin/3.6.3_windows' + } + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + + JAVAURL = 'git@hydra:tools/openjdk.git' + if (isUnix()) { + JAVABRANCH = 'origin/openjdk-11.0.2_linux-x64_bin' + } else { + JAVABRANCH = 'origin/openjdk-11.0.2_windows-x64_bin' + } + JAVADIR = "${WORKSPACE}/java" + configurationhelper.gitCheckoutAndRegister(meta, 'java', JAVAURL, JAVABRANCH, JAVADIR) + JAVA = "${JAVADIR}/ver/bin" + if (isUnix()) { + env.PATH = JAVA + ':' + PATH + } else { + env.PATH = JAVA + ';' + PATH + } + + PPDURL = 'git@hydra:tools/pmd.git' + PPDBRANCH = 'origin/master' + PPDDIR = "${WORKSPACE}/ppd" + configurationhelper.gitCheckoutAndRegister(meta, 'ppd', PPDURL, PPDBRANCH, PPDDIR) + PPD = "${PPDDIR}/bin" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEREVISION = "${SOURCE_REVISION}" + SOURCEDIR = "${WORKSPACE}/source" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + SOURCE_PATH = "${SOURCEDIR}" + SRC = "${SOURCE_PATH}" + if (SOURCE_OFFSET_PATH) { + SRC += "/${SOURCE_OFFSET_PATH}" + } + } + } + } + stage("Clean exports") { + steps { + script { + STATICANALYSIS = 'staticanalysis' + cleanfolders = [ + STATICANALYSIS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.generate(meta, SOURCE_REVISION, SOURCE_PATH) + } + } + } + stage('Static analysis') { + steps { + script { + def PPDFOLDER = "${STATICANALYSIS}/ppd" + def EXCLUDE_PARAM = '' + if (EXCLUDE_PATHS) { + EXCLUDE_PARAM = '--exclude "' + def exclude_path_list = EXCLUDE_PATHS.split(',') + for (exclude_path in exclude_path_list) { + EXCLUDE_PARAM += "${SRC}/${exclude_path.trim()}," + } + EXCLUDE_PARAM += '"' + } + if (isUnix()) { + sh """#!/bin/bash + set -ev + + mkdir -p "${PPDFOLDER}" + export PMD_JAVA_OPTS=-Xmx65536m + "${PPD}/run.sh" cpd --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" + """ + } else { + bat """ + mkdir -p "${PPDFOLDER}" + set PMD_JAVA_OPTS=-Xmx2048m + "${PPD}/cpd.bat" --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" + """ + } + recordIssues(tools: [ + cpd(pattern: "**/cpd.xml", reportEncoding: 'UTF-8'), + ]) + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + ] + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] + def artifactlist = meta.generateartifacts(itemsForRelease) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy new file mode 100644 index 0000000..d959770 --- /dev/null +++ b/staticanalysis/cppcheck.groovy @@ -0,0 +1,114 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + PYTHONBRANCH = 'origin/3.6.3_windows' + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEREVISION = "${SOURCE_REVISION}" + SOURCEDIR = "${WORKSPACE}/source" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + SOURCE_PATH = "${SOURCEDIR}" + env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" + } + } + } + stage("Clean exports") { + steps { + script { + STATICANALYSIS = 'staticanalysis' + cleanfolders = [ + STATICANALYSIS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.generate(meta, SOURCE_REVISION, SOURCE_PATH) + } + } + } + stage('Static analysis') { + steps { + script { + def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" + dir(CPPCHECKFOLDER) { + echo "Executing: ${CPPCHECK} --xml --xml-version=2 ${SRC} 2> cppcheck.xml" + if (isUnix()) { + sh """#!/bin/bash + set -e + + "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml + """ + } else { + bat """ + "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml + """ + } + } + recordIssues(tools: [ + cppCheck(pattern: "**/cppcheck.xml", reportEncoding: 'UTF-8'), + ]) + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + ] + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] + def artifactlist = meta.generateartifacts(itemsForRelease) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/test/testethernetproduct.groovy b/test/testethernetproduct.groovy new file mode 100644 index 0000000..5e2ce53 --- /dev/null +++ b/test/testethernetproduct.groovy @@ -0,0 +1,204 @@ +import jenkins.model.Jenkins + +pipeline { + agent { label AGENT_LABEL } + stages { + stage ("Initial Setup") { + steps{ + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage ("Checkout test environment") { + steps { + script { + toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") + + def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' + def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' + def TEST_ENVIRONMENT_REVISION = 'origin/master' + + def environmentoptions = toolsimporter.getNewToolOptions() + environmentoptions.toolName('Ethernet product test environment') + environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) + + TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" + def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") + TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] + echo "Selected environment:\n${TEST_ENVIRONMENT}" + } + } + } + stage ('Setup') { + agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } + stages { + stage ("Checkout dependencies") { + steps { + script { + def PYTHON_URL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHON_REVISION = 'origin/3.6.3_linux' + } + else { + PYTHON_REVISION = 'origin/3.8.10_windows_for_test' + } + PYTHONDIR = "${WORKSPACE}/python" + + def pythonimportoptions = toolsimporter.getNewToolOptions() + pythonimportoptions.toolName('Python') + pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) + toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + + def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' + TEST_SUITE_DIR = 'TestSuiteDir' + env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" + + def regressiontestimportoptions = toolsimporter.getNewToolOptions() + regressiontestimportoptions.toolName('Tests') + regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) + + env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" + + def resultimportoptions = toolsimporter.getNewToolOptions() + resultimportoptions.toolName('Test history') + resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) + toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) + + def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' + env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" + + def coreimportoptions = toolsimporter.getNewToolOptions() + coreimportoptions.toolName('Test core') + coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) + toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) + + def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' + def FTDI_SOURCE_REVISION = 'origin/Core-98' + FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" + + def ftdiimportoptions = toolsimporter.getNewToolOptions() + ftdiimportoptions.toolName('FTDI relay driver') + ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) + toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) + + def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' + HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" + + def resultprocessingoptions = toolsimporter.getNewToolOptions() + resultprocessingoptions.toolName('History processing') + resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) + toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) + } + } + } + stage ("Import application to test") { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + APPLICATION_UNDER_TEST = 'application_under_test' + env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" + + def applicationimportoptions = importer.getNewImportOptions() + applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) + applicationimportoptions.setfilter('app_package/**,control_package/**') + applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) + + meta.productinfo.set(applicationmeta.productinfo) + meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" + meta.gitinfo.set(applicationmeta.gitinfo) + meta.buildstampinfo.set(applicationmeta.buildstampinfo) + } + } + } + stage("Clean") { + steps { + script { + LOGGING_ROOT = 'Logging' + env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" + cleanfolders = [ + GIS_TEST_OUTPUT_LOGGING_ROOT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage ('Test') { + steps { + script { + lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { + if (POWER_CYCLE_HARDWARE == 'true') { + dir(FTDI_SOURCE_DIR) { + bat """ + "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True + """ + } + } + if (UPDATE_HARDWARE == 'true') { + dir(GIS_TEST_SUITE_CORE) { + bat """ + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" + """ + rc = bat (script:""" + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" + """, + returnStatus: true) + if (rc == 1) { + unstable('FAILED while inspecting state of hardware') + } + } + } + if (DO_TEST == 'true') { + rc = bat(script:""" + ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED while running tests') + } + } + } + } + } + } + stage("Archive Test History") { + steps{ + script { + dir(HISTORY_FOLDER) { + rc = bat (script:""" + "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED to process history') + } + } + } + } + } + stage("Archive test artifacts"){ + steps{ + script { + def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + } + } +} + diff --git a/textutils/subst.groovy b/textutils/subst.groovy new file mode 100644 index 0000000..c0207a3 --- /dev/null +++ b/textutils/subst.groovy @@ -0,0 +1,8 @@ + +def substitute(String filepath, pattern, String subst) { + fileText = readFile(file: filepath, encoding: 'UTF-8') + def converted = fileText.replaceAll(pattern, subst) + writeFile(file: filepath, text: converted, encoding: 'UTF-8') +} + +return this; \ No newline at end of file diff --git a/tool configurations/vs2017install/buildmachines.vsconfig b/tool configurations/vs2017install/buildmachines.vsconfig new file mode 100644 index 0000000..ea618a3 --- /dev/null +++ b/tool configurations/vs2017install/buildmachines.vsconfig @@ -0,0 +1,25 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Workload.ManagedDesktop", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.Workload.NetWeb", + "Microsoft.VisualStudio.Workload.NetCoreTools", + "microsoft.net.componentgroup.targetingpacks.common", + "microsoft.componentgroup.blend", + "microsoft.visualstudio.component.entityframework", + "microsoft.visualstudio.component.debugger.justintime", + "microsoft.visualstudio.component.wcf.tooling", + "microsoft.visualstudio.component.vc.diagnostictools", + "microsoft.visualstudio.component.vc.cmake.project", + "microsoft.visualstudio.component.vc.testadapterforboosttest", + "microsoft.visualstudio.component.vc.testadapterforgoogletest", + "microsoft.visualstudio.component.vc.atlmfc", + "microsoft.visualstudio.component.appinsights.tools", + "microsoft.visualstudio.component.diagnostictools", + "microsoft.visualstudio.componentgroup.web.cloudtools", + "microsoft.visualstudio.component.aspnet45", + "microsoft.visualstudio.component.webdeploy", + "microsoft.visualstudio.component.nuget.buildtools" + ] +} \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy new file mode 100644 index 0000000..a22054b --- /dev/null +++ b/versioncontrol/git.groovy @@ -0,0 +1,377 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") + +def boolean isShallowClonedRepo(String gitrootpath) { + def boolean isshallowclone = true + + dir( gitrootpath) { + if (isUnix()) { + isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() + } + else { + isshallowclone = powershell(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() + } + echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" + } + return isshallowclone +} + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git rev-parse HEAD') + } +} + +ArrayList gettaglist(String workingSetRootPath) { + ArrayList taglist = new ArrayList() + dir(workingSetRootPath) { + def tagliststring = clh.getCommandOutput('git tag --sort=-version:refname') + taglist = tagliststring.split('\\n') + } + return taglist +} + +@NonCPS +def ArrayList sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return sparsecheckoutpaths +} + +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } else { + def reporootname = uniquereporootname(gitURL) + reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootsubfolder}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootsubfolder}" +/* + dir(reporootsubfolder) { + if (isUnix()) { + sh """ + git status + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + powershell """ + git status + if (\$lastexitcode -ge 1) { exit 1 } + """ + } + } +*/ + } + } + return reporootsubfolder +} + +def checkgitlfsinstall() { + if (isUnix()) { + sh """ + git lfs install + if [ \$? -ne 0 ]; then exit 1; fi + """ + + } else { + powershell """ + git lfs install + if (\$lastexitcode -ge 1) { exit 1 } + """ + } +} + +//No credentials specified +//Cloning the remote Git repository +//Cloning repository git@hydra:print-server-releases/print_server_whole.git +//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 +//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git +//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// No valid HEAD. Skipping the resetting +// > git clean -fdx # timeout=10 +// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_7" +// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 + + +// > git rev-parse --is-inside-work-tree # timeout=10 +// Fetching changes from the remote Git repository +// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// Resetting working tree +// > git reset --hard # timeout=10 +// > git clean -fdx # timeout=10 +// Using shallow fetch with depth 1 +// Fetching upstream changes from git@hydra:releases/print-server-installer.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 +// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 +// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git read-tree -mu HEAD # timeout=10 +// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_3" +// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 + +def String lfsinstallandreporoot(String gitURL) { + checkgitlfsinstall() + return checkouttoreporoot(gitURL) +} + +def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { + if (!sparseitemlist.isEmpty()) { + def itemstring = "Items in sparse item list:\n" + for (item in sparseitemlist) { + itemstring += item + "\n" + } + echo itemstring + } +} + +def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { + echo "Git: Checking out: URL: ${gitURL} Revision: ${revision} to: ${targetDir}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'PruneStaleBranch'], + ], + submoduleCfg: [], + ]) +} + +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + +def gitinit(String targetDir, String reporootpath) { + if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + echo "git init ${targetDir}" + if (isUnix()) { + sh """ + git init "${targetDir}" + """ + } else { + powershell """ + git init "${targetDir}" + """ + } + } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") + } +} + +def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { + def String sparseitemliststring = "" + if (!sparseitemlist.isEmpty()) { + for (item in sparseitemlist) { + sparseitemliststring += "${item}," + } + echo "git config lfs.fetchinclude ${sparseitemliststring}" + dir(targetDir) { + if (isUnix()) { + sh """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } else { + powershell """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } + } + } +} + +def gitsparserestore(String targetDir) { + echo "Clean out full fetch; Force a sparse fetch" + dir(targetDir) { + if (isUnix()) { + sh """ + git rm -rf . > /dev/null + git clean -fxd > /dev/null + git reset --hard > /dev/null + """ + } else { + powershell """ + git rm -rf . > \$null + git clean -fxd > \$null + git reset --hard > \$null + """ + } + } + } + +def List convertSparseItemsString(String itemstopublish) { + def List sparsepaths = [] + + boolean nonempty = itemstopublish?.trim() + if (nonempty) { + def String[] items = itemstopublish.trim().split(',') + def String allpaths = "" + for(String item : items) { + sparsepaths.add(item) + allpaths += "${item}\n" + } + echo "Selected item to publish:\n${allpaths}" + } + return sparsepaths +} + +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + echo "Performing alternative checkout" + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitinit(targetDir, reporootpath) + setlfsfetchinclude(targetDir, sparseitemlist) + } + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitsparserestore(targetDir) + } +} + +def checkRepoExists(String gitURL) { + def int result = 0 + if (isUnix()) { + result = sh( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + """, + returnStatus: true + ) + } else { + result = powershell( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + """, + returnStatus: true + ) + } + echo "git ls-remote return result: ${result}" + if (result > 0) { + error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } else { + echo "The release repo: ${gitURL} exists" + } +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } + else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +def stripgithistory(String rootfolder) { + echo "Stripping git history out of ${rootfolder}" + dir(rootfolder) { + if (isUnix()) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + find . -iname "*@tmp" | xargs -iXX rm -rf XX + """ + } else { + echo "Not supporting git stripping in Windows at the moment" + } + } +} + +return this diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..208a60f --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,23 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + def filetoconvert = convertdetail[0] + def pattern = convertdetail[1] + def replacement = convertdetail[2] + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" + echo "Using python: ${PYTHON} and script: ${stampscript}" + if (isUnix()) { + sh """ + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + powershell """ + & "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + } +} + +return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..5e974e2 --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,180 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} + +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" + return newrevision +} + +def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl + + def boolean isShallowCloned = gh.isShallowClonedRepo(gitrootpath) + if (isShallowCloned) { + error("Cannot generate a version number from a shallow cloned repo at ${gitrootpath}") + } + + dir( gitrootpath) { + def int changecount = 0 + if (isUnix()) { + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() + } + else { + changecount = powershell(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() + } + echo "Change count: ${changecount}" + def int mvnp1 = maxversionnumber() + 1 + basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber + adjustedchangecount = incrementrevision(changecount, basebuildnumber) + echo "Adjusted change count: ${adjustedchangecount}" + if (isUnix()) { + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + } + else { + gitrevision = powershell(returnStdout: true, script: "git rev-parse HEAD").trim() + } + echo "Revision: ${gitrevision}" + if (isUnix()) { + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + } + else { + remoteinfo = powershell(returnStdout: true, script: "git remote show origin").trim() + } + echo "Remote info: ${remoteinfo}" + if (isUnix()) { + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + else { + remoteurl = powershell(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + echo "Remote URL: ${remoteurl}" + } + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] +} + +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + + def defaultversionprefix = "0.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } + + def mastertypeversionprefix = "0.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + + def splits2 = sourcebranch.split('\\.') + if (splits2.size() < 1) { + def thirdpartyversionprefix = "0.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + + for (split in splits2) { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + } + + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + if (integerstring.length() > 6) { + integerstring = integerstring.substring(0, 5) + } + int integer = integerstring.toInteger() + integer = integer % (maxversionnumber() + 1) + integers.add(integer) + } + + def versionprefix = "" + if (integers.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { + break + } + } + if (integernumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." + } + } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} + +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) + + def versionprefix = getVersionPrefix(sourcerevision) + + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" + + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) + + return buildstamp +} + +String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { + generate(meta, sourcerevision, sourcepath, basebuildnumber) + meta.buildstampinfo.override(pregeneratedbuildstamp) + + return pregeneratedbuildstamp +} + +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { + def int bbn = basebuildnumber.trim().toInteger() + return generate(meta, sourcerevision, sourcepath, bbn) +} + +String copyfrom(meta, othermeta) { + // Deep copy + meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) + meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) + + return meta.buildstampinfo.buildstamp +} + +return this; \ No newline at end of file diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..9243aa4 --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,38 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting instances of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + with open(filetoconvert,'r') as c: + fileread = c.read() + + count = 0 + with open(filetoconvert, 'w') as f: + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) + f.write(converted) + + if count == 0: + return 1 + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 2c6a90d2e1fa515cdd3ec19d961d595e1a1e0be8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 19:13:16 +0000 Subject: [PATCH 04/72] New files --- product/foreach.groovy | 16 ++++++--- product/gatherreleases.groovy | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 product/gatherreleases.groovy diff --git a/product/foreach.groovy b/product/foreach.groovy index 690c5e1..6d31d01 100644 --- a/product/foreach.groovy +++ b/product/foreach.groovy @@ -51,11 +51,17 @@ pipeline { steps { script { def productcollection = get() + def productreleaseinfo = [] for (productrepo in productcollection.ProductReleaseRepos) { + def query_product_job = build job: QUERY_PRODUCT_REPO_JOB, + parameters: [ + string(name: 'PRODUCT_RELEASE_REPO', value: productrepo), + string(name: 'OUTPUT_ROOT_URL', value: OUTPUT_ROOT_URL), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] } } - } - } -} -} -PRODUCT_COLLECTION \ No newline at end of file + } + } + } +} \ No newline at end of file diff --git a/product/gatherreleases.groovy b/product/gatherreleases.groovy new file mode 100644 index 0000000..6d31d01 --- /dev/null +++ b/product/gatherreleases.groovy @@ -0,0 +1,67 @@ +def productcollectionstore = null + +/* Example expected JSON +{ + "ProductReleaseRepos": [ + "git@hydra:print-server-releases/print_server_whole.git" + ] +}*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: productcollectionstore, pretty: 1) +} + +def boolean exists() { + return (productcollectionstore != null) +} + +def set(String jsontext) { + productcollectionstore = readJSON(text: jsontext) + echo "Product collection:\n${asJsonText()}" +} + +def get() { + return productcollectionstore +} + +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + + set(PRODUCT_COLLECTION) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Iterate product collection') { + steps { + script { + def productcollection = get() + def productreleaseinfo = [] + for (productrepo in productcollection.ProductReleaseRepos) { + def query_product_job = build job: QUERY_PRODUCT_REPO_JOB, + parameters: [ + string(name: 'PRODUCT_RELEASE_REPO', value: productrepo), + string(name: 'OUTPUT_ROOT_URL', value: OUTPUT_ROOT_URL), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + } + } +} \ No newline at end of file From a5ee31d24b52bfdf7147328ec7dc0516d8c71f64 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 19:39:48 +0000 Subject: [PATCH 05/72] New files --- product/foreach.groovy | 4 ++-- product/gatherreleases.groovy | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/product/foreach.groovy b/product/foreach.groovy index 6d31d01..6b95005 100644 --- a/product/foreach.groovy +++ b/product/foreach.groovy @@ -30,8 +30,8 @@ pipeline { stage('Initial setup') { steps { script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) + autobuilder = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilder.startnonbuild(this) set(PRODUCT_COLLECTION) } diff --git a/product/gatherreleases.groovy b/product/gatherreleases.groovy index 6d31d01..6b95005 100644 --- a/product/gatherreleases.groovy +++ b/product/gatherreleases.groovy @@ -30,8 +30,8 @@ pipeline { stage('Initial setup') { steps { script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) + autobuilder = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilder.startnonbuild(this) set(PRODUCT_COLLECTION) } From 2767e8a5c5c3750b6fdbf142a298fd40f81ff10d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 19:44:32 +0000 Subject: [PATCH 06/72] Rename folder --- {Commons_scripts => Common_scripts}/.project | 0 {Commons_scripts => Common_scripts}/.pydevproject | 0 .../.settings/org.eclipse.core.resources.prefs | 0 {Commons_scripts => Common_scripts}/artifacts/archiver.groovy | 0 {Commons_scripts => Common_scripts}/artifacts/metaexport.groovy | 0 {Commons_scripts => Common_scripts}/autobuilder.groovy | 0 {Commons_scripts => Common_scripts}/build/integrity.groovy | 0 {Commons_scripts => Common_scripts}/build/postactions.groovy | 0 {Commons_scripts => Common_scripts}/build/postactionstest.groovy | 0 {Commons_scripts => Common_scripts}/check/build_meta.groovy | 0 {Commons_scripts => Common_scripts}/check/do_meta.groovy | 0 {Commons_scripts => Common_scripts}/checksum/utils.groovy | 0 {Commons_scripts => Common_scripts}/clean/clean.groovy | 0 .../commandline/commandline.groovy | 0 .../compile/eclipsebuild/disableparallelbuilds.groovy | 0 .../compile/eclipsebuild/eclipseSTM8build.groovy | 0 .../compile/eclipsebuild/eclipseds5build.groovy | 0 .../compile/eclipsebuild/eclipseniosbuild.groovy | 0 .../configuration/configuration.groovy | 0 {Commons_scripts => Common_scripts}/configuration/tools.groovy | 0 {Commons_scripts => Common_scripts}/copy/all.groovy | 0 {Commons_scripts => Common_scripts}/copy/esw.groovy | 0 {Commons_scripts => Common_scripts}/copy/filtered.groovy | 0 {Commons_scripts => Common_scripts}/copy/logic.groovy | 0 {Commons_scripts => Common_scripts}/differences/.project | 0 {Commons_scripts => Common_scripts}/differences/.pydevproject | 0 .../differences/dometahierarchy.groovy | 0 {Commons_scripts => Common_scripts}/differences/file.groovy | 0 {Commons_scripts => Common_scripts}/differences/json.groovy | 0 .../differences/metahierarchy.groovy | 0 .../documentation/doxygen_deploy.groovy | 0 {Commons_scripts => Common_scripts}/hardware/sominfo.groovy | 0 {Commons_scripts => Common_scripts}/import/importer.groovy | 0 {Commons_scripts => Common_scripts}/install/installhelper.groovy | 0 {Commons_scripts => Common_scripts}/meta/folder.groovy | 0 {Commons_scripts => Common_scripts}/packaging/debian.groovy | 0 {Commons_scripts => Common_scripts}/platform/lineendings.groovy | 0 {Commons_scripts => Common_scripts}/publish/.classpath | 0 {Commons_scripts => Common_scripts}/publish/.gitignore | 0 {Commons_scripts => Common_scripts}/publish/.project | 0 .../publish/.settings/org.eclipse.jdt.groovy.core.prefs | 0 {Commons_scripts => Common_scripts}/publish/do.groovy | 0 {Commons_scripts => Common_scripts}/publish/package.groovy | 0 .../publish/promptingpublish.groovy | 0 {Commons_scripts => Common_scripts}/publish/publish.groovy | 0 {Commons_scripts => Common_scripts}/publish/publishitems.groovy | 0 {Commons_scripts => Common_scripts}/publishing/utils.groovy | 0 {Commons_scripts => Common_scripts}/release/do.groovy | 0 {Commons_scripts => Common_scripts}/release/registerrepo.groovy | 0 {Commons_scripts => Common_scripts}/release/release.groovy | 0 {Commons_scripts => Common_scripts}/remoteaccess/ftp.groovy | 0 {Commons_scripts => Common_scripts}/remoteaccess/hostedftp.groovy | 0 {Commons_scripts => Common_scripts}/staticanalysis/ccm.groovy | 0 {Commons_scripts => Common_scripts}/staticanalysis/cpd.groovy | 0 .../staticanalysis/cppcheck.groovy | 0 .../test/testethernetproduct.groovy | 0 {Commons_scripts => Common_scripts}/textutils/subst.groovy | 0 .../tool configurations/vs2017install/buildmachines.vsconfig | 0 {Commons_scripts => Common_scripts}/versioncontrol/git.groovy | 0 {Commons_scripts => Common_scripts}/versionnumbering/do.groovy | 0 .../versionnumbering/generate.groovy | 0 {Commons_scripts => Common_scripts}/versionstamping/.project | 0 {Commons_scripts => Common_scripts}/versionstamping/.pydevproject | 0 .../versionstamping/versionstamper.py | 0 64 files changed, 0 insertions(+), 0 deletions(-) rename {Commons_scripts => Common_scripts}/.project (100%) rename {Commons_scripts => Common_scripts}/.pydevproject (100%) rename {Commons_scripts => Common_scripts}/.settings/org.eclipse.core.resources.prefs (100%) rename {Commons_scripts => Common_scripts}/artifacts/archiver.groovy (100%) rename {Commons_scripts => Common_scripts}/artifacts/metaexport.groovy (100%) rename {Commons_scripts => Common_scripts}/autobuilder.groovy (100%) rename {Commons_scripts => Common_scripts}/build/integrity.groovy (100%) rename {Commons_scripts => Common_scripts}/build/postactions.groovy (100%) rename {Commons_scripts => Common_scripts}/build/postactionstest.groovy (100%) rename {Commons_scripts => Common_scripts}/check/build_meta.groovy (100%) rename {Commons_scripts => Common_scripts}/check/do_meta.groovy (100%) rename {Commons_scripts => Common_scripts}/checksum/utils.groovy (100%) rename {Commons_scripts => Common_scripts}/clean/clean.groovy (100%) rename {Commons_scripts => Common_scripts}/commandline/commandline.groovy (100%) rename {Commons_scripts => Common_scripts}/compile/eclipsebuild/disableparallelbuilds.groovy (100%) rename {Commons_scripts => Common_scripts}/compile/eclipsebuild/eclipseSTM8build.groovy (100%) rename {Commons_scripts => Common_scripts}/compile/eclipsebuild/eclipseds5build.groovy (100%) rename {Commons_scripts => Common_scripts}/compile/eclipsebuild/eclipseniosbuild.groovy (100%) rename {Commons_scripts => Common_scripts}/configuration/configuration.groovy (100%) rename {Commons_scripts => Common_scripts}/configuration/tools.groovy (100%) rename {Commons_scripts => Common_scripts}/copy/all.groovy (100%) rename {Commons_scripts => Common_scripts}/copy/esw.groovy (100%) rename {Commons_scripts => Common_scripts}/copy/filtered.groovy (100%) rename {Commons_scripts => Common_scripts}/copy/logic.groovy (100%) rename {Commons_scripts => Common_scripts}/differences/.project (100%) rename {Commons_scripts => Common_scripts}/differences/.pydevproject (100%) rename {Commons_scripts => Common_scripts}/differences/dometahierarchy.groovy (100%) rename {Commons_scripts => Common_scripts}/differences/file.groovy (100%) rename {Commons_scripts => Common_scripts}/differences/json.groovy (100%) rename {Commons_scripts => Common_scripts}/differences/metahierarchy.groovy (100%) rename {Commons_scripts => Common_scripts}/documentation/doxygen_deploy.groovy (100%) rename {Commons_scripts => Common_scripts}/hardware/sominfo.groovy (100%) rename {Commons_scripts => Common_scripts}/import/importer.groovy (100%) rename {Commons_scripts => Common_scripts}/install/installhelper.groovy (100%) rename {Commons_scripts => Common_scripts}/meta/folder.groovy (100%) rename {Commons_scripts => Common_scripts}/packaging/debian.groovy (100%) rename {Commons_scripts => Common_scripts}/platform/lineendings.groovy (100%) rename {Commons_scripts => Common_scripts}/publish/.classpath (100%) rename {Commons_scripts => Common_scripts}/publish/.gitignore (100%) rename {Commons_scripts => Common_scripts}/publish/.project (100%) rename {Commons_scripts => Common_scripts}/publish/.settings/org.eclipse.jdt.groovy.core.prefs (100%) rename {Commons_scripts => Common_scripts}/publish/do.groovy (100%) rename {Commons_scripts => Common_scripts}/publish/package.groovy (100%) rename {Commons_scripts => Common_scripts}/publish/promptingpublish.groovy (100%) rename {Commons_scripts => Common_scripts}/publish/publish.groovy (100%) rename {Commons_scripts => Common_scripts}/publish/publishitems.groovy (100%) rename {Commons_scripts => Common_scripts}/publishing/utils.groovy (100%) rename {Commons_scripts => Common_scripts}/release/do.groovy (100%) rename {Commons_scripts => Common_scripts}/release/registerrepo.groovy (100%) rename {Commons_scripts => Common_scripts}/release/release.groovy (100%) rename {Commons_scripts => Common_scripts}/remoteaccess/ftp.groovy (100%) rename {Commons_scripts => Common_scripts}/remoteaccess/hostedftp.groovy (100%) rename {Commons_scripts => Common_scripts}/staticanalysis/ccm.groovy (100%) rename {Commons_scripts => Common_scripts}/staticanalysis/cpd.groovy (100%) rename {Commons_scripts => Common_scripts}/staticanalysis/cppcheck.groovy (100%) rename {Commons_scripts => Common_scripts}/test/testethernetproduct.groovy (100%) rename {Commons_scripts => Common_scripts}/textutils/subst.groovy (100%) rename {Commons_scripts => Common_scripts}/tool configurations/vs2017install/buildmachines.vsconfig (100%) rename {Commons_scripts => Common_scripts}/versioncontrol/git.groovy (100%) rename {Commons_scripts => Common_scripts}/versionnumbering/do.groovy (100%) rename {Commons_scripts => Common_scripts}/versionnumbering/generate.groovy (100%) rename {Commons_scripts => Common_scripts}/versionstamping/.project (100%) rename {Commons_scripts => Common_scripts}/versionstamping/.pydevproject (100%) rename {Commons_scripts => Common_scripts}/versionstamping/versionstamper.py (100%) diff --git a/Commons_scripts/.project b/Common_scripts/.project similarity index 100% rename from Commons_scripts/.project rename to Common_scripts/.project diff --git a/Commons_scripts/.pydevproject b/Common_scripts/.pydevproject similarity index 100% rename from Commons_scripts/.pydevproject rename to Common_scripts/.pydevproject diff --git a/Commons_scripts/.settings/org.eclipse.core.resources.prefs b/Common_scripts/.settings/org.eclipse.core.resources.prefs similarity index 100% rename from Commons_scripts/.settings/org.eclipse.core.resources.prefs rename to Common_scripts/.settings/org.eclipse.core.resources.prefs diff --git a/Commons_scripts/artifacts/archiver.groovy b/Common_scripts/artifacts/archiver.groovy similarity index 100% rename from Commons_scripts/artifacts/archiver.groovy rename to Common_scripts/artifacts/archiver.groovy diff --git a/Commons_scripts/artifacts/metaexport.groovy b/Common_scripts/artifacts/metaexport.groovy similarity index 100% rename from Commons_scripts/artifacts/metaexport.groovy rename to Common_scripts/artifacts/metaexport.groovy diff --git a/Commons_scripts/autobuilder.groovy b/Common_scripts/autobuilder.groovy similarity index 100% rename from Commons_scripts/autobuilder.groovy rename to Common_scripts/autobuilder.groovy diff --git a/Commons_scripts/build/integrity.groovy b/Common_scripts/build/integrity.groovy similarity index 100% rename from Commons_scripts/build/integrity.groovy rename to Common_scripts/build/integrity.groovy diff --git a/Commons_scripts/build/postactions.groovy b/Common_scripts/build/postactions.groovy similarity index 100% rename from Commons_scripts/build/postactions.groovy rename to Common_scripts/build/postactions.groovy diff --git a/Commons_scripts/build/postactionstest.groovy b/Common_scripts/build/postactionstest.groovy similarity index 100% rename from Commons_scripts/build/postactionstest.groovy rename to Common_scripts/build/postactionstest.groovy diff --git a/Commons_scripts/check/build_meta.groovy b/Common_scripts/check/build_meta.groovy similarity index 100% rename from Commons_scripts/check/build_meta.groovy rename to Common_scripts/check/build_meta.groovy diff --git a/Commons_scripts/check/do_meta.groovy b/Common_scripts/check/do_meta.groovy similarity index 100% rename from Commons_scripts/check/do_meta.groovy rename to Common_scripts/check/do_meta.groovy diff --git a/Commons_scripts/checksum/utils.groovy b/Common_scripts/checksum/utils.groovy similarity index 100% rename from Commons_scripts/checksum/utils.groovy rename to Common_scripts/checksum/utils.groovy diff --git a/Commons_scripts/clean/clean.groovy b/Common_scripts/clean/clean.groovy similarity index 100% rename from Commons_scripts/clean/clean.groovy rename to Common_scripts/clean/clean.groovy diff --git a/Commons_scripts/commandline/commandline.groovy b/Common_scripts/commandline/commandline.groovy similarity index 100% rename from Commons_scripts/commandline/commandline.groovy rename to Common_scripts/commandline/commandline.groovy diff --git a/Commons_scripts/compile/eclipsebuild/disableparallelbuilds.groovy b/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy similarity index 100% rename from Commons_scripts/compile/eclipsebuild/disableparallelbuilds.groovy rename to Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy diff --git a/Commons_scripts/compile/eclipsebuild/eclipseSTM8build.groovy b/Common_scripts/compile/eclipsebuild/eclipseSTM8build.groovy similarity index 100% rename from Commons_scripts/compile/eclipsebuild/eclipseSTM8build.groovy rename to Common_scripts/compile/eclipsebuild/eclipseSTM8build.groovy diff --git a/Commons_scripts/compile/eclipsebuild/eclipseds5build.groovy b/Common_scripts/compile/eclipsebuild/eclipseds5build.groovy similarity index 100% rename from Commons_scripts/compile/eclipsebuild/eclipseds5build.groovy rename to Common_scripts/compile/eclipsebuild/eclipseds5build.groovy diff --git a/Commons_scripts/compile/eclipsebuild/eclipseniosbuild.groovy b/Common_scripts/compile/eclipsebuild/eclipseniosbuild.groovy similarity index 100% rename from Commons_scripts/compile/eclipsebuild/eclipseniosbuild.groovy rename to Common_scripts/compile/eclipsebuild/eclipseniosbuild.groovy diff --git a/Commons_scripts/configuration/configuration.groovy b/Common_scripts/configuration/configuration.groovy similarity index 100% rename from Commons_scripts/configuration/configuration.groovy rename to Common_scripts/configuration/configuration.groovy diff --git a/Commons_scripts/configuration/tools.groovy b/Common_scripts/configuration/tools.groovy similarity index 100% rename from Commons_scripts/configuration/tools.groovy rename to Common_scripts/configuration/tools.groovy diff --git a/Commons_scripts/copy/all.groovy b/Common_scripts/copy/all.groovy similarity index 100% rename from Commons_scripts/copy/all.groovy rename to Common_scripts/copy/all.groovy diff --git a/Commons_scripts/copy/esw.groovy b/Common_scripts/copy/esw.groovy similarity index 100% rename from Commons_scripts/copy/esw.groovy rename to Common_scripts/copy/esw.groovy diff --git a/Commons_scripts/copy/filtered.groovy b/Common_scripts/copy/filtered.groovy similarity index 100% rename from Commons_scripts/copy/filtered.groovy rename to Common_scripts/copy/filtered.groovy diff --git a/Commons_scripts/copy/logic.groovy b/Common_scripts/copy/logic.groovy similarity index 100% rename from Commons_scripts/copy/logic.groovy rename to Common_scripts/copy/logic.groovy diff --git a/Commons_scripts/differences/.project b/Common_scripts/differences/.project similarity index 100% rename from Commons_scripts/differences/.project rename to Common_scripts/differences/.project diff --git a/Commons_scripts/differences/.pydevproject b/Common_scripts/differences/.pydevproject similarity index 100% rename from Commons_scripts/differences/.pydevproject rename to Common_scripts/differences/.pydevproject diff --git a/Commons_scripts/differences/dometahierarchy.groovy b/Common_scripts/differences/dometahierarchy.groovy similarity index 100% rename from Commons_scripts/differences/dometahierarchy.groovy rename to Common_scripts/differences/dometahierarchy.groovy diff --git a/Commons_scripts/differences/file.groovy b/Common_scripts/differences/file.groovy similarity index 100% rename from Commons_scripts/differences/file.groovy rename to Common_scripts/differences/file.groovy diff --git a/Commons_scripts/differences/json.groovy b/Common_scripts/differences/json.groovy similarity index 100% rename from Commons_scripts/differences/json.groovy rename to Common_scripts/differences/json.groovy diff --git a/Commons_scripts/differences/metahierarchy.groovy b/Common_scripts/differences/metahierarchy.groovy similarity index 100% rename from Commons_scripts/differences/metahierarchy.groovy rename to Common_scripts/differences/metahierarchy.groovy diff --git a/Commons_scripts/documentation/doxygen_deploy.groovy b/Common_scripts/documentation/doxygen_deploy.groovy similarity index 100% rename from Commons_scripts/documentation/doxygen_deploy.groovy rename to Common_scripts/documentation/doxygen_deploy.groovy diff --git a/Commons_scripts/hardware/sominfo.groovy b/Common_scripts/hardware/sominfo.groovy similarity index 100% rename from Commons_scripts/hardware/sominfo.groovy rename to Common_scripts/hardware/sominfo.groovy diff --git a/Commons_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy similarity index 100% rename from Commons_scripts/import/importer.groovy rename to Common_scripts/import/importer.groovy diff --git a/Commons_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy similarity index 100% rename from Commons_scripts/install/installhelper.groovy rename to Common_scripts/install/installhelper.groovy diff --git a/Commons_scripts/meta/folder.groovy b/Common_scripts/meta/folder.groovy similarity index 100% rename from Commons_scripts/meta/folder.groovy rename to Common_scripts/meta/folder.groovy diff --git a/Commons_scripts/packaging/debian.groovy b/Common_scripts/packaging/debian.groovy similarity index 100% rename from Commons_scripts/packaging/debian.groovy rename to Common_scripts/packaging/debian.groovy diff --git a/Commons_scripts/platform/lineendings.groovy b/Common_scripts/platform/lineendings.groovy similarity index 100% rename from Commons_scripts/platform/lineendings.groovy rename to Common_scripts/platform/lineendings.groovy diff --git a/Commons_scripts/publish/.classpath b/Common_scripts/publish/.classpath similarity index 100% rename from Commons_scripts/publish/.classpath rename to Common_scripts/publish/.classpath diff --git a/Commons_scripts/publish/.gitignore b/Common_scripts/publish/.gitignore similarity index 100% rename from Commons_scripts/publish/.gitignore rename to Common_scripts/publish/.gitignore diff --git a/Commons_scripts/publish/.project b/Common_scripts/publish/.project similarity index 100% rename from Commons_scripts/publish/.project rename to Common_scripts/publish/.project diff --git a/Commons_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs similarity index 100% rename from Commons_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs rename to Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs diff --git a/Commons_scripts/publish/do.groovy b/Common_scripts/publish/do.groovy similarity index 100% rename from Commons_scripts/publish/do.groovy rename to Common_scripts/publish/do.groovy diff --git a/Commons_scripts/publish/package.groovy b/Common_scripts/publish/package.groovy similarity index 100% rename from Commons_scripts/publish/package.groovy rename to Common_scripts/publish/package.groovy diff --git a/Commons_scripts/publish/promptingpublish.groovy b/Common_scripts/publish/promptingpublish.groovy similarity index 100% rename from Commons_scripts/publish/promptingpublish.groovy rename to Common_scripts/publish/promptingpublish.groovy diff --git a/Commons_scripts/publish/publish.groovy b/Common_scripts/publish/publish.groovy similarity index 100% rename from Commons_scripts/publish/publish.groovy rename to Common_scripts/publish/publish.groovy diff --git a/Commons_scripts/publish/publishitems.groovy b/Common_scripts/publish/publishitems.groovy similarity index 100% rename from Commons_scripts/publish/publishitems.groovy rename to Common_scripts/publish/publishitems.groovy diff --git a/Commons_scripts/publishing/utils.groovy b/Common_scripts/publishing/utils.groovy similarity index 100% rename from Commons_scripts/publishing/utils.groovy rename to Common_scripts/publishing/utils.groovy diff --git a/Commons_scripts/release/do.groovy b/Common_scripts/release/do.groovy similarity index 100% rename from Commons_scripts/release/do.groovy rename to Common_scripts/release/do.groovy diff --git a/Commons_scripts/release/registerrepo.groovy b/Common_scripts/release/registerrepo.groovy similarity index 100% rename from Commons_scripts/release/registerrepo.groovy rename to Common_scripts/release/registerrepo.groovy diff --git a/Commons_scripts/release/release.groovy b/Common_scripts/release/release.groovy similarity index 100% rename from Commons_scripts/release/release.groovy rename to Common_scripts/release/release.groovy diff --git a/Commons_scripts/remoteaccess/ftp.groovy b/Common_scripts/remoteaccess/ftp.groovy similarity index 100% rename from Commons_scripts/remoteaccess/ftp.groovy rename to Common_scripts/remoteaccess/ftp.groovy diff --git a/Commons_scripts/remoteaccess/hostedftp.groovy b/Common_scripts/remoteaccess/hostedftp.groovy similarity index 100% rename from Commons_scripts/remoteaccess/hostedftp.groovy rename to Common_scripts/remoteaccess/hostedftp.groovy diff --git a/Commons_scripts/staticanalysis/ccm.groovy b/Common_scripts/staticanalysis/ccm.groovy similarity index 100% rename from Commons_scripts/staticanalysis/ccm.groovy rename to Common_scripts/staticanalysis/ccm.groovy diff --git a/Commons_scripts/staticanalysis/cpd.groovy b/Common_scripts/staticanalysis/cpd.groovy similarity index 100% rename from Commons_scripts/staticanalysis/cpd.groovy rename to Common_scripts/staticanalysis/cpd.groovy diff --git a/Commons_scripts/staticanalysis/cppcheck.groovy b/Common_scripts/staticanalysis/cppcheck.groovy similarity index 100% rename from Commons_scripts/staticanalysis/cppcheck.groovy rename to Common_scripts/staticanalysis/cppcheck.groovy diff --git a/Commons_scripts/test/testethernetproduct.groovy b/Common_scripts/test/testethernetproduct.groovy similarity index 100% rename from Commons_scripts/test/testethernetproduct.groovy rename to Common_scripts/test/testethernetproduct.groovy diff --git a/Commons_scripts/textutils/subst.groovy b/Common_scripts/textutils/subst.groovy similarity index 100% rename from Commons_scripts/textutils/subst.groovy rename to Common_scripts/textutils/subst.groovy diff --git a/Commons_scripts/tool configurations/vs2017install/buildmachines.vsconfig b/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig similarity index 100% rename from Commons_scripts/tool configurations/vs2017install/buildmachines.vsconfig rename to Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig diff --git a/Commons_scripts/versioncontrol/git.groovy b/Common_scripts/versioncontrol/git.groovy similarity index 100% rename from Commons_scripts/versioncontrol/git.groovy rename to Common_scripts/versioncontrol/git.groovy diff --git a/Commons_scripts/versionnumbering/do.groovy b/Common_scripts/versionnumbering/do.groovy similarity index 100% rename from Commons_scripts/versionnumbering/do.groovy rename to Common_scripts/versionnumbering/do.groovy diff --git a/Commons_scripts/versionnumbering/generate.groovy b/Common_scripts/versionnumbering/generate.groovy similarity index 100% rename from Commons_scripts/versionnumbering/generate.groovy rename to Common_scripts/versionnumbering/generate.groovy diff --git a/Commons_scripts/versionstamping/.project b/Common_scripts/versionstamping/.project similarity index 100% rename from Commons_scripts/versionstamping/.project rename to Common_scripts/versionstamping/.project diff --git a/Commons_scripts/versionstamping/.pydevproject b/Common_scripts/versionstamping/.pydevproject similarity index 100% rename from Commons_scripts/versionstamping/.pydevproject rename to Common_scripts/versionstamping/.pydevproject diff --git a/Commons_scripts/versionstamping/versionstamper.py b/Common_scripts/versionstamping/versionstamper.py similarity index 100% rename from Commons_scripts/versionstamping/versionstamper.py rename to Common_scripts/versionstamping/versionstamper.py From 9a31beb1d7dd099ef5292d234a686cdbb0dfdd84 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 13 Feb 2022 19:45:52 +0000 Subject: [PATCH 07/72] Remove folder --- Common_scripts/.project | 17 - Common_scripts/.pydevproject | 8 - .../org.eclipse.core.resources.prefs | 2 - Common_scripts/artifacts/archiver.groovy | 17 - Common_scripts/artifacts/metaexport.groovy | 26 - Common_scripts/autobuilder.groovy | 147 -- Common_scripts/build/integrity.groovy | 113 -- Common_scripts/build/postactions.groovy | 225 ---- Common_scripts/build/postactionstest.groovy | 43 - Common_scripts/check/build_meta.groovy | 87 -- Common_scripts/check/do_meta.groovy | 77 -- Common_scripts/checksum/utils.groovy | 15 - Common_scripts/clean/clean.groovy | 10 - Common_scripts/commandline/commandline.groovy | 12 - .../eclipsebuild/disableparallelbuilds.groovy | 12 - .../eclipsebuild/eclipseSTM8build.groovy | 172 --- .../eclipsebuild/eclipseds5build.groovy | 252 ---- .../eclipsebuild/eclipseniosbuild.groovy | 116 -- .../configuration/configuration.groovy | 18 - Common_scripts/configuration/tools.groovy | 59 - Common_scripts/copy/all.groovy | 67 - Common_scripts/copy/esw.groovy | 67 - Common_scripts/copy/filtered.groovy | 73 - Common_scripts/copy/logic.groovy | 79 -- Common_scripts/differences/.project | 17 - Common_scripts/differences/.pydevproject | 8 - .../differences/dometahierarchy.groovy | 836 ------------ Common_scripts/differences/file.groovy | 24 - Common_scripts/differences/json.groovy | 51 - .../differences/metahierarchy.groovy | 102 -- .../documentation/doxygen_deploy.groovy | 80 -- Common_scripts/hardware/sominfo.groovy | 80 -- Common_scripts/import/importer.groovy | 408 ------ Common_scripts/install/installhelper.groovy | 136 -- Common_scripts/meta/folder.groovy | 1193 ----------------- Common_scripts/packaging/debian.groovy | 111 -- Common_scripts/platform/lineendings.groovy | 5 - Common_scripts/publish/.classpath | 9 - Common_scripts/publish/.gitignore | 7 - Common_scripts/publish/.project | 18 - .../org.eclipse.jdt.groovy.core.prefs | 3 - Common_scripts/publish/do.groovy | 174 --- Common_scripts/publish/package.groovy | 181 --- .../publish/promptingpublish.groovy | 182 --- Common_scripts/publish/publish.groovy | 92 -- Common_scripts/publish/publishitems.groovy | 129 -- Common_scripts/publishing/utils.groovy | 27 - Common_scripts/release/do.groovy | 458 ------- Common_scripts/release/registerrepo.groovy | 12 - Common_scripts/release/release.groovy | 87 -- Common_scripts/remoteaccess/ftp.groovy | 32 - Common_scripts/remoteaccess/hostedftp.groovy | 28 - Common_scripts/staticanalysis/ccm.groovy | 85 -- Common_scripts/staticanalysis/cpd.groovy | 152 --- Common_scripts/staticanalysis/cppcheck.groovy | 114 -- .../test/testethernetproduct.groovy | 204 --- Common_scripts/textutils/subst.groovy | 8 - .../vs2017install/buildmachines.vsconfig | 25 - Common_scripts/versioncontrol/git.groovy | 377 ------ Common_scripts/versionnumbering/do.groovy | 23 - .../versionnumbering/generate.groovy | 180 --- Common_scripts/versionstamping/.project | 17 - Common_scripts/versionstamping/.pydevproject | 8 - .../versionstamping/versionstamper.py | 38 - 64 files changed, 7435 deletions(-) delete mode 100644 Common_scripts/.project delete mode 100644 Common_scripts/.pydevproject delete mode 100644 Common_scripts/.settings/org.eclipse.core.resources.prefs delete mode 100644 Common_scripts/artifacts/archiver.groovy delete mode 100644 Common_scripts/artifacts/metaexport.groovy delete mode 100644 Common_scripts/autobuilder.groovy delete mode 100644 Common_scripts/build/integrity.groovy delete mode 100644 Common_scripts/build/postactions.groovy delete mode 100644 Common_scripts/build/postactionstest.groovy delete mode 100644 Common_scripts/check/build_meta.groovy delete mode 100644 Common_scripts/check/do_meta.groovy delete mode 100644 Common_scripts/checksum/utils.groovy delete mode 100644 Common_scripts/clean/clean.groovy delete mode 100644 Common_scripts/commandline/commandline.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/eclipseSTM8build.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/eclipseds5build.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/eclipseniosbuild.groovy delete mode 100644 Common_scripts/configuration/configuration.groovy delete mode 100644 Common_scripts/configuration/tools.groovy delete mode 100644 Common_scripts/copy/all.groovy delete mode 100644 Common_scripts/copy/esw.groovy delete mode 100644 Common_scripts/copy/filtered.groovy delete mode 100644 Common_scripts/copy/logic.groovy delete mode 100644 Common_scripts/differences/.project delete mode 100644 Common_scripts/differences/.pydevproject delete mode 100644 Common_scripts/differences/dometahierarchy.groovy delete mode 100644 Common_scripts/differences/file.groovy delete mode 100644 Common_scripts/differences/json.groovy delete mode 100644 Common_scripts/differences/metahierarchy.groovy delete mode 100644 Common_scripts/documentation/doxygen_deploy.groovy delete mode 100644 Common_scripts/hardware/sominfo.groovy delete mode 100644 Common_scripts/import/importer.groovy delete mode 100644 Common_scripts/install/installhelper.groovy delete mode 100644 Common_scripts/meta/folder.groovy delete mode 100644 Common_scripts/packaging/debian.groovy delete mode 100644 Common_scripts/platform/lineendings.groovy delete mode 100644 Common_scripts/publish/.classpath delete mode 100644 Common_scripts/publish/.gitignore delete mode 100644 Common_scripts/publish/.project delete mode 100644 Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs delete mode 100644 Common_scripts/publish/do.groovy delete mode 100644 Common_scripts/publish/package.groovy delete mode 100644 Common_scripts/publish/promptingpublish.groovy delete mode 100644 Common_scripts/publish/publish.groovy delete mode 100644 Common_scripts/publish/publishitems.groovy delete mode 100644 Common_scripts/publishing/utils.groovy delete mode 100644 Common_scripts/release/do.groovy delete mode 100644 Common_scripts/release/registerrepo.groovy delete mode 100644 Common_scripts/release/release.groovy delete mode 100644 Common_scripts/remoteaccess/ftp.groovy delete mode 100644 Common_scripts/remoteaccess/hostedftp.groovy delete mode 100644 Common_scripts/staticanalysis/ccm.groovy delete mode 100644 Common_scripts/staticanalysis/cpd.groovy delete mode 100644 Common_scripts/staticanalysis/cppcheck.groovy delete mode 100644 Common_scripts/test/testethernetproduct.groovy delete mode 100644 Common_scripts/textutils/subst.groovy delete mode 100644 Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig delete mode 100644 Common_scripts/versioncontrol/git.groovy delete mode 100644 Common_scripts/versionnumbering/do.groovy delete mode 100644 Common_scripts/versionnumbering/generate.groovy delete mode 100644 Common_scripts/versionstamping/.project delete mode 100644 Common_scripts/versionstamping/.pydevproject delete mode 100644 Common_scripts/versionstamping/versionstamper.py diff --git a/Common_scripts/.project b/Common_scripts/.project deleted file mode 100644 index e329c3f..0000000 --- a/Common_scripts/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - Common Build Scripts - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/Common_scripts/.pydevproject b/Common_scripts/.pydevproject deleted file mode 100644 index 1ce5929..0000000 --- a/Common_scripts/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - - -/${PROJECT_DIR_NAME} - -python 2.7 -python35 - diff --git a/Common_scripts/.settings/org.eclipse.core.resources.prefs b/Common_scripts/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 1174a06..0000000 --- a/Common_scripts/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,2 +0,0 @@ -eclipse.preferences.version=1 -encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/Common_scripts/artifacts/archiver.groovy b/Common_scripts/artifacts/archiver.groovy deleted file mode 100644 index bafba02..0000000 --- a/Common_scripts/artifacts/archiver.groovy +++ /dev/null @@ -1,17 +0,0 @@ -enum Store { - JENKINS, - ARTIFACTSERVER, -} - -this.JENKINS = Store.JENKINS -this.ARTIFACTSERVER = Store.JENKINS - -def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { - echo "Archiving artifacts to store: ${store}" - meta.copyrelatedmeta() - if (store == JENKINS) { - archiveArtifacts(artifacts: artifactlist) - } -} - -return this \ No newline at end of file diff --git a/Common_scripts/artifacts/metaexport.groovy b/Common_scripts/artifacts/metaexport.groovy deleted file mode 100644 index a40e9a8..0000000 --- a/Common_scripts/artifacts/metaexport.groovy +++ /dev/null @@ -1,26 +0,0 @@ -// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(meta, importer, subJobs) { - def ArrayList artifactFolders = new ArrayList() - for (subJob in subJobs) { - def subJobObject = subJob[0] - if (subJobObject != null) { - def subJobName = subJobObject.getFullProjectName() - def subJobNumber = subJobObject.getNumber() - def destFolder = subJob[1] - def boolean incrementversionnumber = subJob[2] - dir(destFolder) { - deleteDir() - } - - def importoptions = importer.getNewImportOptions() - importoptions.jenkinsjob(subJobName, subJobNumber) - importoptions.setincrementversionnumber(incrementversionnumber) - importer.artifactimport(meta, importoptions, destFolder) - - artifactFolders.add("${destFolder}/**") - } - } - return artifactFolders -} - -return this; diff --git a/Common_scripts/autobuilder.groovy b/Common_scripts/autobuilder.groovy deleted file mode 100644 index 667f89a..0000000 --- a/Common_scripts/autobuilder.groovy +++ /dev/null @@ -1,147 +0,0 @@ -// Assume that WORKSPACE environment variable is set - -autobuildroot = 'autobuild' -commonscriptsroot = "${autobuildroot}/Common_scripts" -metafolderscript = null - -def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { - if (doclean) { - echo "Cleaning workspace" - // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - def stashname = 'Clean workspace stash' - stash(allowEmpty: true, - includes: "${autobuildroot}/**,${autobuildroot}/.git/**", - name: stashname, - useDefaultExcludes: false - ) - cleanWs() - unstash name: stashname - } - else { - echo "Not cleaning workspace" - } -} - -def commonscripts() { - return commonscriptsroot -} - -def getautobuildpath() { - return autobuildroot -} - -def getmetafolderscript() { - return metafolderscript -} - -def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { - def String altered = releaserepo.trim() - - if (releaserepovariantsuffix?.trim()) { - altered = altered.replace("-RELEASE_REPO_VARIANT_SUFFIX-", releaserepovariantsuffix) - } - return altered -} - -def setBuildType(meta, boolean dofullbuild = true) { - def doquickbuild = false - - echo "Build trigger cause: ${currentBuild.getBuildCauses()}" - if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { - echo "Build triggered by user" - if (!dofullbuild) { - echo "Marking as quick build as user wanted one" - doquickbuild = true - } - } else { - echo "Build triggered automatically" - if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { - echo "Marking as quick build as it was triggered by a commit" - doquickbuild = true - } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { - echo "Marking as full build as it was triggered by timer" - } - } - echo "Build type is ${doquickbuild ? "quick" : "full"}" - if (doquickbuild) { - unstable('This not a full build - a subset of the artifacts is not being generated') - } - meta.jenkinsbuildinfo.set(!doquickbuild) - return doquickbuild -} - -def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { - def meta = getmetafolderscript().metareset(script) - echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" - meta.productinfo.set(productname, subcomponentname, productvariant) - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - for (p in params) { - meta.jenkinsbuildinfo.addParameter(p.key, p.value) - } - return meta -} - -// Start up boilerplate -def startbuild( - script, - String productname = PRODUCT_NAME, - String subcomponentname = SUBCOMPONENT_NAME, - String releaserepo = RELEASE_REPO, - String productvariant = '', - String autobuildpath = autobuildroot, - String commonscriptspath = commonscriptsroot - ) { - echo "Starting...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - metafolderscript = load("${commonscripts()}/meta/folder.groovy") - - conditionalclean() - - def meta = setupmeta(script, productname, subcomponentname, productvariant) - - def rr = load("${commonscripts()}/release/registerrepo.groovy") - def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) - - echo "Release repo: ${alteredreleaserepo}" - rr.registerrepo(meta, alteredreleaserepo) - - def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration(meta) - - return meta -} - -// Start up boilerplate -def startnonbuild( - script, - autobuildpath = autobuildroot, - commonscriptspath = commonscriptsroot - ) { - echo "Starting non-build job...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - metafolderscript = load("${commonscripts()}/meta/folder.groovy") - - conditionalclean() - - def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") - - def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration(meta) - - return meta -} - -def configurationhelper() { - return load("${commonscripts()}/configuration/configuration.groovy") -} - -// Everyone wants to know this -autobuilder = this - -return this; diff --git a/Common_scripts/build/integrity.groovy b/Common_scripts/build/integrity.groovy deleted file mode 100644 index e93c00f..0000000 --- a/Common_scripts/build/integrity.groovy +++ /dev/null @@ -1,113 +0,0 @@ -import static groovy.json.JsonOutput.* - -def check(buildmeta) { - def results = [ - 'good': true, - 'comment': '', - ] - - // Walk the meta looking for different versions of the same components - def allmeta = buildmeta.allrelatedmeta() - echo "All meta:\n${allmeta.comment}" - - // Store build stamps associated with each product component; Store a counter against each product component version - def productcomponents = [:] - for (meta in allmeta.metacollection) { - def key = meta.componentidentifier() - - if (!productcomponents.containsKey(key)) { - productcomponents[key] = [:] - } - def buildstampnumber = meta.releasetag() - if (!productcomponents[key].containsKey(buildstampnumber)) { - productcomponents[key][buildstampnumber] = 1 - } else { - productcomponents[key][buildstampnumber] += 1 - } - } - - // Show what was collected - echo prettyPrint(toJson(productcomponents)) - - // Look for product components with more than one build stamp - for (pc in productcomponents.keySet()) { - def bsns = productcomponents[pc] - if (bsns.size() > 1) { - results.good = false - results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" - for (bsn in bsns.keySet()) { - results.comment += "${bsn}\n" - } - } - } - - // Store product components associated with each release git url - def releasegiturls = [:] - for (meta in allmeta.metacollection) { - def String key = meta.releaseinfo.releaserepoURL - - if (key) { - if (!releasegiturls.containsKey(key)) { - releasegiturls[key] = [:] - } - def componentidentifier = meta.componentidentifier() - if (!releasegiturls[key].containsKey(componentidentifier)) { - releasegiturls[key][componentidentifier] = 1 - } else { - releasegiturls[key][componentidentifier] += 1 - } - } - } - - // Show what was collected - echo prettyPrint(toJson(releasegiturls)) - - // Look for release git urls being used by more than one product component - for (rgu in releasegiturls.keySet()) { - def cis = releasegiturls[rgu] - if (cis.size() > 1) { - results.good = false - results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" - for (ci in cis.keySet()) { - results.comment += "${ci}\n" - } - } - } - - // Store git commit shas associated with each source git url - def sourcegiturls = [:] - for (meta in allmeta.metacollection) { - def String key = meta.gitinfo.remoteurl - - if (key) { - if (!sourcegiturls.containsKey(key)) { - sourcegiturls[key] = [:] - } - def commitsha = meta.gitinfo.gitrevision - if (!sourcegiturls[key].containsKey(commitsha)) { - sourcegiturls[key][commitsha] = 1 - } else { - sourcegiturls[key][commitsha] += 1 - } - } - } - - // Show what was collected - echo prettyPrint(toJson(sourcegiturls)) - - // Look for source git urls associated with more than one sha code - for (sgu in sourcegiturls.keySet()) { - def cs = sourcegiturls[sgu] - if (cs.size() > 1) { - results.good = false - results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" - for (c in cs.keySet()) { - results.comment += "${c}\n" - } - } - } - - return results -} - -return this \ No newline at end of file diff --git a/Common_scripts/build/postactions.groovy b/Common_scripts/build/postactions.groovy deleted file mode 100644 index 8b18908..0000000 --- a/Common_scripts/build/postactions.groovy +++ /dev/null @@ -1,225 +0,0 @@ -/* Example post build info -{ - "Build outcomes": { - "Success": { - "Actions": { - "Deploy documentation": { - "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", - "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" - }, - "Static analysis": { - "Description": "Can be empty - Static analysis job to run when this build is successful", - "Job": "" - }, - "Unit test": { - "Description": "Can be empty - Unit test job to run when this build is successful", - "Job": "" - }, - "Regression test": { - "Description": "Can be empty - Regression test job to run when this build is successful", - "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" - }, - "Update differences": { - "Description": "Can be empty - Update differences job to run when this build is successful", - "Job": "HMB_FD_HV_diff_releases" - }, - "Release to git": { - "Description": "Can be empty - Name of release to git job to run if this build is successful", - "Job": "HMB_FD_HV_release_to_git" - } - } - }, - "Failed": { - "Actions": { - "Email": { - "Description": "Can be empty - Name of release to git job to run if this build is successful", - "To": [ - "dan.hawkins@globalinkjetsystems.com" - ] - }, - "ExtEmail": { - "Description": "Send an email using Extended Email plugin", - "To": [ - "dan.hawkins@globalinkjetsystems.com" - ], - "Recipient providers": [ - "culprits", - "requestor", - "developers", - "upstreamDevelopers" - ] - } - } - } - } -} -*/ - -def getActionJobName(postbuildactions, String action, String alternativeparamname) { - def String jobname = '' - if (postbuildactions) { - try { - jobname = postbuildactions[action]['Job'] - } catch(Exception ex) { - echo "No post build actions found for action: ${action}" - jobname = '' - } - } else { - if (params[alternativeparamname] != null) { - if (params[alternativeparamname]) { - jobname = params[alternativeparamname] - } - } - } - if (jobname) { - def jobitem = Jenkins.instance.getItemByFullName(jobname) - if (jobitem) { - if (!jobitem.isBuildable()) { - jobname = '' - } - } else { - jobname = '' - } - } - // echo "Job name for action: ${action}: ${jobname}" - return jobname -} - -def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { - def jobname = getActionJobName(postbuildactions, action, alternativeparamname) - if (jobname) { - build( - wait: false, - job: jobname, - parameters: [ - string(name: jobnameparametername, value: JOB_NAME), - string(name: jobselectorparametername, value: buildselector), - ] - ) - } -} - -def readPostBuildInfo() { - def postbuildinfo = readJSON(text: POST_BUILD_INFO) - def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) - // echo "Post build info:\n${postbuildjsontext}" - - return postbuildinfo -} - -def performSuccessfulBuildActions() { - def postbuildactions - if (params.POST_BUILD_INFO != null) { - def postbuildinfo = readPostBuildInfo() - try { - postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] - } catch(Exception ex) { - echo "Post build info did not contain: Build outcomes/Success/Actions" - postbuildactions = null - } - } - def String buildselector = "${BUILD_NUMBER}" - deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') - deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) - deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') - deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') -} - -def sendemails(emailinfo) { - def emailtolist = emailinfo['To'] - - if (emailtolist) { - def emailbody = """ - Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} - - Console output: ${BUILD_URL}console - """ - for(emailto in emailtolist) { - mail( - subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", - body: emailbody, - to: emailto - ) - // echo "Send email to: ${emailto}" - } - } -} - -def getRecipientProviders(recipientProviderInfoList) { - def recipientProviderMap = [ - culprits: [$class: 'CulpritsRecipientProvider'], - requestor: [$class: 'RequesterRecipientProvider'], - developers: [$class: 'DevelopersRecipientProvider'], - upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], - ] - def recipientProviders = [] - - if (recipientProviderInfoList) { - for(rp in recipientProviderInfoList) { - if (recipientProviderMap.containsKey(rp)) { - recipientProviders.add(recipientProviderMap[rp]) - } - } - } - if (recipientProviders.size() == 0) { - recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] - } - // echo "recipientProviders: ${recipientProviders.toString()}" - return recipientProviders; -} - -def String getToList(toListInfo) { - def String toList = '' - - if (toListInfo) { - toList = toListInfo.join(',') - } - // echo "toList: ${toList}" - return toList -} - -def sendextemail(extmailinfo) { - def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) - def toList = getToList(extmailinfo['To']) - - emailext( - subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", - body: '''${JELLY_SCRIPT,template="html"}''', - attachLog: true, - // attachmentsPattern: '', - compressLog: false, - // from : '', - mimeType: 'text/html', - // postsendScript: '', - // presendScript: '', - recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers - // replyTo: '', - saveOutput: false, - to: toList, - ) -} - -def performFailedBuildActions() { - def postbuildactions = null - if (params.POST_BUILD_INFO != null) { - def postbuildinfo = readPostBuildInfo() - try { - postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] - } catch(Exception ex) { - echo "Post build info did not contain: Build outcomes/Failed/Actions" - postbuildactions = null - } - } - if (postbuildactions) { - if (postbuildactions['Email']) { - sendemails(postbuildactions['Email']) - } - if (postbuildactions['ExtEmail']) { - sendextemail(postbuildactions['ExtEmail']) - } - } -} - -return this; diff --git a/Common_scripts/build/postactionstest.groovy b/Common_scripts/build/postactionstest.groovy deleted file mode 100644 index 150f5d1..0000000 --- a/Common_scripts/build/postactionstest.groovy +++ /dev/null @@ -1,43 +0,0 @@ -pipeline { - agent { label 'master' } - stages { - stage('Initial setup') { - steps { - script { - CLEAN_WORKSPACE = false - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Dummy build job') { - steps { - script { - echo "Execute dummy build job: ${CAUSE_ERROR}" - if (CAUSE_ERROR == 'true') { - error("Causing error") - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - script { - load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() - } - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - script { - load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() - } - } - } -} diff --git a/Common_scripts/check/build_meta.groovy b/Common_scripts/check/build_meta.groovy deleted file mode 100644 index fb8a75d..0000000 --- a/Common_scripts/check/build_meta.groovy +++ /dev/null @@ -1,87 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - cleanfolders = [ - "${WORKSPACE}/build_artifacts", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Get build artifact meta') { - steps { - script { - BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" - copyArtifacts( - projectName: BUILD_JOB, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR - ], - target: BUILD_ARTIFACTS - ) - def mfs = autobuilderscript.getmetafolderscript() - buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) - } - } - } - stage('Check components') { - steps { - script { - def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") - (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/check/do_meta.groovy b/Common_scripts/check/do_meta.groovy deleted file mode 100644 index 4b8a89e..0000000 --- a/Common_scripts/check/do_meta.groovy +++ /dev/null @@ -1,77 +0,0 @@ -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" -releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" -tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Release summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason += reason + "\n" - echo "Marked build as unstable: Reason: ${reason}" -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { - int md = maxdepth.toInteger() - resetfolders() - preconditions() - - // Walk the meta information to a max depth - def buildinfo = buildmeta.releasewalker(md) - echo "All components:" - def String componentinfo = "" - for (entry in buildinfo) { - def name = entry.metainfo.componentversionuniqueidentifier() - def offsetpath = entry.accumulatedpath - def abspath = "${buildpathroot}/${offsetpath}" - componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" - } - echo componentinfo - - // Walk the meta information to a max depth - def componentReleaseInfo = buildmeta.releasewalkerunique(md) - echo "Unique components:" - def String versioncomponent = "" - for (key in componentReleaseInfo.keySet()) { - def buildinfo2 = componentReleaseInfo[key] - def offsetpath = buildinfo2.accumulatedpath - def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" - } - echo versioncomponent - - return [summarystring, markunstable, unstablereason] -} - -return this; diff --git a/Common_scripts/checksum/utils.groovy b/Common_scripts/checksum/utils.groovy deleted file mode 100644 index be7569d..0000000 --- a/Common_scripts/checksum/utils.groovy +++ /dev/null @@ -1,15 +0,0 @@ - -// Generate a checksum file of from a list of files/directories relative to workspace -def generatelist(meta, checksummable) { - for (filepath in checksummable) { - def checksumset = meta.checksuminfo.addChecksumSet(filepath) - def files = findFiles(glob: filepath) - for (file in files) { - echo "Checksumming: ${file.path}" - def sum = sha1(file: file.path).trim() - checksumset.addChecksumPair(file.path, sum) - } - } -} - -return this; \ No newline at end of file diff --git a/Common_scripts/clean/clean.groovy b/Common_scripts/clean/clean.groovy deleted file mode 100644 index 4707ba0..0000000 --- a/Common_scripts/clean/clean.groovy +++ /dev/null @@ -1,10 +0,0 @@ - -def clean(folderlist) { - for (folder in folderlist) { - dir(folder) { - deleteDir() - } - } -} - -return this; \ No newline at end of file diff --git a/Common_scripts/commandline/commandline.groovy b/Common_scripts/commandline/commandline.groovy deleted file mode 100644 index 38f80fa..0000000 --- a/Common_scripts/commandline/commandline.groovy +++ /dev/null @@ -1,12 +0,0 @@ -String getCommandOutput(String cmd) { - if (isUnix()) { - return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() - } else { - return powershell(returnStdout: true, script: """ - ${cmd} - """ - ).trim() - } -} - -return this \ No newline at end of file diff --git a/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy b/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy deleted file mode 100644 index 03e4c8f..0000000 --- a/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy +++ /dev/null @@ -1,12 +0,0 @@ - -// Disable parallel make builds - which may be causing autobuild issues -def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { - if (isUnix()) { - sh """ - # Looks for " sparseitemlist = gh.convertSparseItemsString(filter) - gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - - return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) -} - -return this diff --git a/Common_scripts/configuration/tools.groovy b/Common_scripts/configuration/tools.groovy deleted file mode 100644 index dbaeedb..0000000 --- a/Common_scripts/configuration/tools.groovy +++ /dev/null @@ -1,59 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -class ToolOptions { - def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - - String m_toolName - - String m_gitURL - String m_revision - String m_repoRoot - int m_cloneDepth - boolean m_shallowClone - - ToolOptions(script) { - m_script = script - m_gitURL = '-' - m_revision = '-' - m_repoRoot = '' - m_cloneDepth = 1 - m_shallowClone = true - } - - void toolName(String name) { - m_toolName = name.trim() - } - - void toolRepo(String gitURL, String revision) { - m_gitURL = gitURL.trim() - m_revision = revision.trim() - } - - void repoRoot(String root) { - m_repoRoot = root.trim() - } - - void clonedepth(int depth) { - m_cloneDepth = depth - } - - void shallowclone(boolean shallow) { - m_shallowClone = shallow - } -} - -ToolOptions getNewToolOptions() { - return new ToolOptions(this) -} - -def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) -} - -def getTool(meta, ToolOptions options, String targetDir) { - gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) - - registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) -} - -return this diff --git a/Common_scripts/copy/all.groovy b/Common_scripts/copy/all.groovy deleted file mode 100644 index 24aacfb..0000000 --- a/Common_scripts/copy/all.groovy +++ /dev/null @@ -1,67 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - // Force workspace to be cleaned - CLEAN_WORKSPACE = "true" - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - } - } - } - stage('Import source artifacts') { - steps { - script { - def copyoptions = importer.getNewImportOptions() - copyoptions.source( - SOURCE_ARTIFACT_LOCATION, - SOURCE_ARTIFACT_BUILD_JOB, - SOURCE_ARTIFACT_BUILD_SELECTOR, - SOURCE_ARTIFACT_RELEASE_REPO, - SOURCE_ARTIFACT_RELEASE_REVISION) - echo "Nested: ${NESTED_COPY}" - copyoptions.setnested(NESTED_COPY == 'true') - importer.artifactimport(meta, copyoptions, "${WORKSPACE}/.") - } - } - } - stage('Archiving artifacts') { - steps { - script { - // Save the imported meta - while the unwanted meta is written to file then deleted - def stashname = 'Imported meta stash' - stash(allowEmpty: true, - includes: 'meta/**', - name: stashname, - useDefaultExcludes: false) - - def artifactlist = meta.generateartifacts(['**/*']) - cleaner.clean(['meta', 'autobuild']) - - unstash(name: stashname) - - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/copy/esw.groovy b/Common_scripts/copy/esw.groovy deleted file mode 100644 index b9eb737..0000000 --- a/Common_scripts/copy/esw.groovy +++ /dev/null @@ -1,67 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean exports') { - steps { - script { - env.ESW = 'esw' - env.ESWDEST = "${WORKSPACE}/${ESW}" - env.ESWEXPORT = 'embedded_software' - cleanfolders = [ - ESWDEST, - ESWEXPORT, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Copying embedded software artifacts') { - steps { - script { - def eswimportoptions = importer.getNewImportOptions() - eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) - eswimportoptions.setincrementversionnumber(true) - importer.artifactimport(meta, eswimportoptions, ESWDEST) - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) - sh """ - rm -v -rf diff - rm -v -rf meta - mv -v "${ESW}"/* . - """ - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} \ No newline at end of file diff --git a/Common_scripts/copy/filtered.groovy b/Common_scripts/copy/filtered.groovy deleted file mode 100644 index 3ad24cd..0000000 --- a/Common_scripts/copy/filtered.groovy +++ /dev/null @@ -1,73 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean') { - steps { - script { - env.IMPORT = "${WORKSPACE}/import" - env.FILTEREDDEST = "${WORKSPACE}/filtered" - cleanfolders = [ - FILTEREDDEST, - IMPORT, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Import artifacts') { - steps { - script { - def importoptions = importer.getNewImportOptions() - importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) - importer.artifactimport(meta, importoptions, IMPORT) - } - } - } - stage('Filtered copy') { - steps { - script { - dir(IMPORT) { - fileOperations([ - fileCopyOperation(excludes: EXCLUDES, - flattenFiles: FLATTENOUTPUT, - includes: INCLUDES, - targetLocation: FILTEREDDEST)]) - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts(["filtered/**"]) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/copy/logic.groovy b/Common_scripts/copy/logic.groovy deleted file mode 100644 index db9dbc3..0000000 --- a/Common_scripts/copy/logic.groovy +++ /dev/null @@ -1,79 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean exports') { - steps { - script { - LOGICDEST = 'logic' - cleanfolders = [ - LOGICDEST, - "${WORKSPACE}/embedded_logic", - "${WORKSPACE}/el", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Import source artifacts') { - steps { - script { - def logicoptions = importer.getNewImportOptions() - logicoptions.source( - LOGIC_ARTIFACT_LOCATION, - LOGIC_ARTIFACT_PROJECT, - LOGIC_ARTIFACT_BUILD_SELECTOR, - LOGIC_ARTIFACT_RELEASE_REPO, - LOGIC_ARTIFACT_RELEASE_REVISION) - importer.artifactimport(meta, logicoptions, LOGICDEST) - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts(['**/*']) - def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - - cleaner.clean(['meta', 'autobuild']) - - dir(LOGICDEST) { - fileOperations([ - fileCopyOperation(excludes: '', - flattenFiles: false, - includes: '**/*', - targetLocation: WORKSPACE) - ]) - } - - cleaner.clean([LOGICDEST]) - - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/differences/.project b/Common_scripts/differences/.project deleted file mode 100644 index 75af0fc..0000000 --- a/Common_scripts/differences/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - Release_difference - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/Common_scripts/differences/.pydevproject b/Common_scripts/differences/.pydevproject deleted file mode 100644 index c3434da..0000000 --- a/Common_scripts/differences/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - - -/${PROJECT_DIR_NAME} - -python 3.0 -Default - diff --git a/Common_scripts/differences/dometahierarchy.groovy b/Common_scripts/differences/dometahierarchy.groovy deleted file mode 100644 index 3b25798..0000000 --- a/Common_scripts/differences/dometahierarchy.groovy +++ /dev/null @@ -1,836 +0,0 @@ -import groovy.json.JsonSlurper - -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" - -sourcerepofolder = "${WORKSPACE}/sourcerepos" -tempfolder = "${WORKSPACE}/temp" - -reportfolder = "${WORKSPACE}/reports" - -class JiraTicketInfo { - String jiranumber - String uri - - JiraTicketInfo() { - jiranumber = '' - uri = '' - } - - def String dump() { - return "jiranumber: ${jiranumber} uri: ${uri}" - } -} - -class DifferenceRecord { - String productname - String subcomponentname - - String rawdifferencetext - String sha - String type - String mergeinfo - String squashinfo - List jiraticketinfo - String comment - - DifferenceRecord() { - productname = "" - subcomponentname = "" - - rawdifferencetext = "" - sha = "" - type = "" - mergeinfo = "" - squashinfo = "" - jiraticketinfo = [] - comment = "" - } - - def String dump() { - String jiraticketstring = '' - for (j in jiraticketinfo) { - jiraticketstring += j.dump() + " " - } - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" - } -} - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } - dir(reportfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Release summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason += reason + "\n" - echo "Marked build as unstable: Reason: ${reason}" -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def showmetaitems(metaitems) { - String info = '' - for (key in metaitems.keySet()) { - def relatedmetainfo = metaitems[key] - def accumulatedpath = relatedmetainfo.accumulatedpath - info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" - } - echo info -} - -def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { - boolean checkedout = false - def String sourcerepo = buildmeta.gitinfo.remoteurl - if (sourcerepo?.trim()) { - echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" - if (sourcerepo) { - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } - } - checkedout = true - } - return checkedout -} - -def String getcomponent(meta) { - def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - - boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) - - return [checkedout, uniquefolderforsourcerepo] -} - -def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { - def String releaseexpression - if (tripledot) { - releaseexpression = earlyrevision + "..." + laterrevision - } else { - releaseexpression = earlyrevision + ".." + laterrevision - } - def String differencetext - dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() - } - return differencetext -} - -def extractsha(String line) { - def pattern = ~"^(\\p{XDigit}*) (.*)" - def match = (line =~ pattern) - match.find() - - def sha = match.group(1) - def restofline = match.group(2) - - //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" - - return [sha.trim(), restofline.trim()] -} - -def extractsquashes(String line) { - def pattern1 = "^(.*)(Squashed .*)" - def match1 = (line =~ pattern1) - if (match1.find()) { - return [true, match1.group(2)] - } - - return [false, line] -} - -def extractmerges(String line) { - def pattern1 = "^(.*)(Merge branch .*)" - def match1 = (line =~ pattern1) - if (match1.find()) { - return [true, match1.group(2)] - } - - def pattern2 = "^(.*)(Merge remote-tracking branch .*)" - def match2 = (line =~ pattern2) - if (match2.find()) { - return [true, match2.group(2)] - } - - def pattern3 = "^(.*)(Merge commit .*)" - def match3 = (line =~ pattern3) - if (match3.find()) { - return [true, match3.group(2)] - } - - def pattern4 = "^(.*)(Add .* from commit .*)" - def match4 = (line =~ pattern4) - if (match4.find()) { - return [true, match4.group(2)] - } - - return [false, line] -} - -def jiraticketexists(String jiraticket) { - def String URI = "${JIRA_URI}/browse/${jiraticket}" - def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" - def get = new URL(RestfulURI).openConnection(); - get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) - get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) - get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) - get.setRequestProperty( 'Accept', 'application/json' ) - def getRC = get.getResponseCode(); - - def String redirectedjiraticket = jiraticket - if(!getRC.equals(200)) { - return [false, '', jiraticket] - } else { - def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) - redirectedjiraticket = jsonresponse['key'] - } - return [true, URI, redirectedjiraticket] -} - -def extractjiraticketinfo(String line) { - def List jiratickets = [] - boolean exists = false - String URI = '' - - while (true) { - line = line.trim() - def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line - def commamatch = (line =~ commapattern) - if (commamatch.find()) { - line = commamatch.group(1).trim() - } - def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 - def match1 = (line =~ pattern1) - if (match1.find()) { - jiraticket = match1.group(1) - (exists, URI, jiraticket) = jiraticketexists(jiraticket) - if (exists) { - def JiraTicketInfo j = new JiraTicketInfo() - j.jiranumber = jiraticket - j.uri = URI - jiratickets.add(j) - line = match1.group(2) - } else { - break - } - } else { - def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 - def match2 = (line =~ pattern2) - if (match2.find()) { - jiraticket = match2.group(1) + "-" + match2.group(2) - (exists, URI, jiraticket) = jiraticketexists(jiraticket) - if (exists) { - def JiraTicketInfo j = new JiraTicketInfo() - j.jiranumber = jiraticket - j.uri = URI - jiratickets.add(j) - line = match2.group(3) - } else { - break - } - } else { - def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 - def match3 = (line =~ pattern3) - if (match3.find()) { - jiraticket = match3.group(1) + "-" + match3.group(2) - (exists, URI, jiraticket) = jiraticketexists(jiraticket) - if (exists) { - def JiraTicketInfo j = new JiraTicketInfo() - j.jiranumber = jiraticket - j.uri = URI - jiratickets.add(j) - line = match3.group(3) - } else { - break - } - } else { - break - } - } - } - } - boolean result = jiratickets.size() > 0 - return [result, jiratickets, line] -} - -def extractreverts(String line) { - boolean result = true - - def pattern1 = '^(.*)Revert \"(.*)\"' - def match1 = (line =~ pattern1) - if (match1.find()) { - def line2 = match1.group(2) - (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) - return [true, originaljiraticketinfo, originalcomment] - } - - def pattern2 = '^(.*)Partial Revert \"(.*)\"' - def match2 = (line =~ pattern2) - if (match2.find()) { - def line2 = match2.group(2) - (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) - return [true, originaljiraticketinfo, originalcomment] - } - - return [false, [], line] -} - -def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { - def String differencetext - dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() - } - return differencetext -} - -def List fillsquashrecords(String sourcepath, String sha) { - def List differencerecords = [] - - String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) - int count = 0 - squashloglines.split('\n').each { - if (count > 0) { // Ignore first line - def line = it - if (line?.trim()) { - def List drs = fillrecord(meta, line.trim(), sourcepath) - differencerecords.addAll(drs) - //echo "${dr.dump()}" - } - } - count += 1 - } - return differencerecords -} - -def List fillrecord(meta, line, String sourcepath) { - def List differencerecords = [] - def DifferenceRecord dr = new DifferenceRecord() - - dr.productname = meta.productinfo.productname - dr.subcomponentname = meta.productinfo.subcomponentname - - dr.rawdifferencetext = line - - // Example of raw text to process: - // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py - - (sha, line2) = extractsha(line) - dr.sha = sha - - (result, squashinfo) = extractsquashes(line2) - if (result) { - dr.type = "Squash" - dr.squashinfo = squashinfo - squashrecords = fillsquashrecords(sourcepath, sha) - differencerecords.addAll(squashrecords) - } else { - (result, mergeinfo) = extractmerges(line2) - if (result) { - dr.type = "Merge" - dr.mergeinfo = mergeinfo - } else { - (result, jiraticketinfo, comment) = extractreverts(line2) - if (result) { - dr.type = "Revert" - dr.jiraticketinfo = jiraticketinfo - dr.comment = comment - } else { - (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) - if (result) { - dr.type = "Ticketed" - dr.jiraticketinfo = jiraticketinfo - dr.comment = comment - } else { - dr.type = "Unticketed" - dr.comment = line2 - } - } - } - } - differencerecords.add(dr) - return differencerecords -} - -def List processtext(meta, String text, String sourcepath) { - def List differencerecords = [] - - text.split('\n').each { - def line = it - if (line?.trim()) { - def List drs = fillrecord(meta, line.trim(), sourcepath) - differencerecords.addAll(drs) - //echo "${dr.dump()}" - } - } - return differencerecords -} - -def getdifferencerecords(earlymeta, latermeta, String sourcepath) { - boolean result = true; - def List differencerecords = [] - - def earlygitinfo = earlymeta.gitinfo - def latergitinfo = latermeta.gitinfo - - if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { - def earlyrevision = earlygitinfo.gitrevision - def laterrevision = latergitinfo.gitrevision - echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" - summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") - def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - differencerecords = processtext(earlymeta, differencetext, sourcepath) - if (earlyrevision == laterrevision) { - summaryappend(" No change") - } - } else { - summaryappend("The early and later git urls don't match") - echo "The early and later git urls don't match" - result = false; - setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") - } - - return [result, differencerecords] -} - -def deletedcomponent(earlymetaitems, key) { - echo "Component ${key} has been deleted in later version" - summaryappend("Deleted ") -} - -def addedcomponent(latermetaitems, key) { - echo "Component ${key} has been added in later version" - summaryappend("Added ") -} - -class SquashReportRecord { - String comment - int duplicates - - SquashReportRecord() { - comment = '' - duplicates = 0 - } -} - -def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { - dir(reportsubfolder) { - def String squashlist = "${title}\nSquash list:\n" - def squashessorted = squashes.sort() - for (asquash in squashessorted) { - def SquashReportRecord s = asquash.value - squashlist += "${s.comment} (${s.duplicates})\n" - } - writeFile(file: "squashlist.txt", text: squashlist) - } -} - -class MergeReportRecord { - String comment - int duplicates - - MergeReportRecord() { - comment = '' - duplicates = 0 - } -} - -def mergereport(String reportsubfolder, String title, LinkedHashMap merges, boolean iscomponent = true) { - dir(reportsubfolder) { - def String mergelist = "${title}\nMerge list:\n" - def mergessorted = merges.sort() - for (amerge in mergessorted) { - def MergeReportRecord m = amerge.value - mergelist += "${m.comment} (${m.duplicates})\n" - } - writeFile(file: "mergelist.txt", text: mergelist) - } -} - -class CommentsRecord { - String comment - int duplicates - - CommentsRecord() { - comment = '' - duplicates = 0 - } -} - -class RevertsReportRecord { - String jiraticket - String ticketURI - LinkedHashMap commentscollection - - RevertsReportRecord() { - jiraticket = '' - ticketURI = '' - commentscollection = new LinkedHashMap() - } -} - -def revertsreport(String reportsubfolder, String title, LinkedHashMap reverts, boolean iscomponent = true) { - dir(reportsubfolder) { - def String revertlist = "${title}\nRevert list:\n" - def revertssorted = reverts.sort() - for (arevert in revertssorted) { - def RevertsReportRecord r = arevert.value - revertlist += "${r.jiraticket} (${r.ticketURI}):\n" - def commentscollection = r.commentscollection - def sortedcommentscollection = commentscollection.sort() - for (commentrecord in sortedcommentscollection) { - def CommentsRecord c = commentrecord.value - revertlist += "${c.comment} (${c.duplicates})\n" - } - revertlist += "\n" - } - writeFile(file: "revertlist.txt", text: revertlist) - } -} - -class TicketedReportRecord { - String jiraticket - String ticketURI - LinkedHashMap commentscollection - - TicketedReportRecord() { - jiraticket = '' - ticketURI = '' - commentscollection = new LinkedHashMap() - } -} - -def ticketedreport(String reportsubfolder, String title, LinkedHashMap ticketed, boolean iscomponent = true) { - dir(reportsubfolder) { - def String ticketedlist = "${title}\nTicketed list:\n" - def ticketedsorted = ticketed.sort() - for (aticketed in ticketedsorted) { - def TicketedReportRecord r = aticketed.value - ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" - def commentscollection = r.commentscollection - def sortedcommentscollection = commentscollection.sort() - for (commentrecord in sortedcommentscollection) { - def CommentsRecord c = commentrecord.value - ticketedlist += "${c.comment} (${c.duplicates})\n" - } - ticketedlist += "\n" - } - writeFile(file: "ticketedlist.txt", text: ticketedlist) - } -} - -class UnticketedReportRecord { - String comment - int duplicates - - UnticketedReportRecord() { - comment = '' - duplicates = 0 - } -} - -def unticketedreport(String reportsubfolder, String title, LinkedHashMap unticketed, boolean iscomponent = true) { - dir(reportsubfolder) { - def String unticketedlist = "${title}\nUnticketed list:\n" - def unticketedsorted = unticketed.sort() - for (anunticketed in unticketedsorted) { - UnticketedReportRecord u = anunticketed.value - unticketedlist += "${u.comment} (${u.duplicates})\n" - } - writeFile(file: "unticketedlist.txt", text: unticketedlist) - } -} - -class TicketsReportRecord { - String jiraticket - String ticketURI - int duplicates - - TicketsReportRecord() { - jiraticket = '' - ticketURI = '' - duplicates = 0 - } -} - -def ticketsreport(String reportsubfolder, String title, tickets, boolean iscomponent = true) { - dir(reportsubfolder) { - String ticketlist = "${title}\nTicket list:\n" - ticketssorted = tickets.sort() - for (ticketnumber in ticketssorted) { - TicketsReportRecord t = ticketnumber.value - ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" - } - writeFile(file: "ticketlist.txt", text: ticketlist) - } -} - -def differencerecordsreport(String reportsubfolder, String title, differencerecords, boolean iscomponent = true) { - dir(reportsubfolder) { - String recordlist = "${title}\n" - for (dr in differencerecords) { - recordlist += dr.dump() + "\n\n" - } - writeFile(file: "recordlist.txt", text: recordlist) - } -} - -def boolean report(meta, title, differencerecords, boolean iscomponent = true) { - def Map ticketed = [:] - def Map squashes = [:] - def Map merges = [:] - def Map reverts = [:] - def Map unticketed = [:] - def Map tickets = [:] - - for (dr in differencerecords) { - if (dr.type == "Squash") { - def SquashReportRecord s - if (!squashes.containsKey(dr.mergeinfo)) { - s = new SquashReportRecord() - s.comment = dr.squashinfo - s.duplicates = 1 - } else { - s = squashes[dr.squashinfo] - s.duplicates += 1 - } - squashes[dr.squashinfo] = s - } - if (dr.type == "Merge") { - def MergeReportRecord m - if (!merges.containsKey(dr.mergeinfo)) { - m = new MergeReportRecord() - m.comment = dr.mergeinfo - m.duplicates = 1 - } else { - m = merges[dr.mergeinfo] - m.duplicates += 1 - } - merges[dr.mergeinfo] = m - } - if (dr.type == "Revert") { - def RevertsReportRecord r - for (ticket in dr.jiraticketinfo) { - JiraTicketInfo j = ticket - if (!reverts.containsKey(j.jiranumber)) { - r = new RevertsReportRecord() - r.jiraticket = j.jiranumber - r.ticketURI = j.uri - CommentsRecord c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - r.commentscollection[dr.comment] = c - } else { - r = reverts[j.jiranumber] - def CommentsRecord c - if (!r.commentscollection.containsKey(dr.comment)) { - c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - r.commentscollection[dr.comment] = c - } else { - c = r.commentscollection[dr.comment] - c.duplicates += 1 - } - r.commentscollection[dr.comment] = c - } - reverts[j.jiranumber] = r - def TicketsReportRecord t - if (!tickets.containsKey(j.jiranumber)) { - t = new TicketsReportRecord() - t.jiraticket = j.jiranumber - t.ticketURI = j.uri - t.duplicates = 1 - } else { - t = tickets[j.jiranumber] - t.duplicates += 1 - } - tickets[j.jiranumber] = t - } - } - if (dr.type == "Ticketed") { - def TicketedReportRecord t - for (ticket in dr.jiraticketinfo) { - JiraTicketInfo j = ticket - if (!ticketed.containsKey(j.jiranumber)) { - t = new TicketedReportRecord() - t.jiraticket = j.jiranumber - t.ticketURI = j.uri - CommentsRecord c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - t.commentscollection[dr.comment] = c - } else { - t = ticketed[j.jiranumber] - def CommentsRecord c - if (!t.commentscollection.containsKey(dr.comment)) { - c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - t.commentscollection[dr.comment] = c - } else { - c = t.commentscollection[dr.comment] - c.duplicates += 1 - } - t.commentscollection[dr.comment] = c - } - ticketed[j.jiranumber] = t - def TicketsReportRecord ti - if (!tickets.containsKey(j.jiranumber)) { - ti = new TicketsReportRecord() - ti.jiraticket = j.jiranumber - ti.ticketURI = j.uri - ti.duplicates = 1 - } else { - ti = tickets[j.jiranumber] - ti.duplicates += 1 - } - tickets[j.jiranumber] = ti - } - } - if (dr.type == "Unticketed") { - UnticketedReportRecord u - if (!unticketed.containsKey(dr.comment)) { - u = new UnticketedReportRecord() - u.comment = dr.comment - u.duplicates = 1 - } else { - u = unticketed[dr.comment] - u.duplicates += 1 - } - unticketed[dr.comment] = u - } - } - String reportsubfolder = "" - if (iscomponent) { - reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" - } else { - reportsubfolder = "${reportfolder}/Combined" - } - mergereport(reportsubfolder, title, merges, iscomponent) - squashreport(reportsubfolder, title, squashes, iscomponent) - revertsreport(reportsubfolder, title, reverts, iscomponent) - ticketedreport(reportsubfolder, title, ticketed, iscomponent) - unticketedreport(reportsubfolder, title, unticketed, iscomponent) - ticketsreport(reportsubfolder, title, tickets, iscomponent) - differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) -} - -def title(earlymeta, latermeta) { - return """ - Differences in: - Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} - between: - then: - Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} - now: - Build date: ${latermeta.jenkinsbuildinfo.builddate} Build stamp: ${latermeta.buildstampinfo.buildstamp} Git revision: ${latermeta.gitinfo.gitrevision} - """ -} - -def compare(earlymetaitems, latermetaitems, key) { - boolean result = true; - echo "Collecting changes between early and later versions of component: ${key}" - - def earlymeta = earlymetaitems[key].metainfo - def latermeta = latermetaitems[key].metainfo - - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") - - def boolean repoavailable - def String uniquefolderforsourcerepo - (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) - - List differencerecords = [] - if (repoavailable) { - (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - - summaryappend(" Items: ${differencerecords.size()} ") - report(earlymeta, title(earlymeta, latermeta), differencerecords) - } else { - summaryappend(" No source repo ") - } - summaryappend("\n") - - return [result, differencerecords]; -} - -def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { - def List differencerecords = [] - - def int md = maxdepth.toInteger() - resetfolders() - preconditions() - - echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - - def earlymeta = meta.importrelated(true, earlymetafolder) - def latermeta = meta.importrelated(true, latermetafolder) - - def earlymetaitems = earlymeta.releasewalkerunique(md, '', earlymetafolder) - def latermetaitems = latermeta.releasewalkerunique(md, '', latermetafolder) - - echo "Early items:" - showmetaitems(earlymetaitems) - - echo "Later items:" - showmetaitems(latermetaitems) - - for (key in earlymetaitems.keySet()) { - if (latermetaitems.containsKey(key)) { - (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) - differencerecords = differencerecords + componentdifferencerecords - } else { - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") - deletedcomponent(earlymetaitems, key) - summaryappend("\n") - } - } - for (key in latermetaitems.keySet()) { - if (!earlymetaitems.containsKey(key)) { - summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") - addedcomponent(latermetaitems, key) - summaryappend("\n") - } - } - report(meta, title(earlymeta, latermeta), differencerecords, false) - return [summarystring, markunstable, unstablereason] -} - -return this; diff --git a/Common_scripts/differences/file.groovy b/Common_scripts/differences/file.groovy deleted file mode 100644 index ee1b8b3..0000000 --- a/Common_scripts/differences/file.groovy +++ /dev/null @@ -1,24 +0,0 @@ -def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { - echo "Comparing file: ${importedfile} with: ${currentfile} option: ${ignorewhitespace}" - def wsparameter = "-w" - if (!ignorewhitespace) { - wsparameter = "" - } - if (isUnix()) { - sh """ - diff -c ${wsparameter} "${importedfile}" "${currentfile}" - """ - } - else { - // diffutils can be installed on windows with chocolatey - powershell """ - # echo Checking utilities exist - # where diff - # if errorlevel 1 EXIT /b 1 - - diff.exe -c ${wsparameter} "${importedfile}" "${currentfile}" - """ - } -} - -return this; diff --git a/Common_scripts/differences/json.groovy b/Common_scripts/differences/json.groovy deleted file mode 100644 index dbf47b8..0000000 --- a/Common_scripts/differences/json.groovy +++ /dev/null @@ -1,51 +0,0 @@ -def comparefiles(importedjsonfile, currentjsonfile) { - echo "Comparing json file: ${importedjsonfile} with: ${currentjsonfile}" - if (isUnix()) { - sh """ - which jq - if [ \$? -ne 0 ]; then exit 1; fi - # jq -S sorts json output - diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") - """ - } - else { - // Jq and diffutils can be installed on windows with chocolatey - powershell """ - # echo Checking utilities exist - # set - # echo %PATH% - try { - if (Get-Command jq) {} - } - catch { - echo "jq is unavailable" - exit \$false - } - try { - if (Get-Command diff) {} - } - catch { - echo "diff is unavailable" - exit \$false - } - - # echo Sorting json in files to compare - # jq -S sorts json output - jq -S . "${importedjsonfile}" > temp1.txt - if (\$lastexitcode -ge 1) { exit 1 } - jq -S . "${currentjsonfile}" > temp2.txt - if (\$lastexitcode -ge 1) { exit 1 } - - # echo Diffing sorted json files - diff.exe -c temp1.txt temp2.txt - \$lec = \$lastexitcode - - Remove-Item temp1.txt - Remove-Item temp2.txt - - if (\$lec -ge 1) { exit 1 } - """ - } -} - -return this; diff --git a/Common_scripts/differences/metahierarchy.groovy b/Common_scripts/differences/metahierarchy.groovy deleted file mode 100644 index 01c1890..0000000 --- a/Common_scripts/differences/metahierarchy.groovy +++ /dev/null @@ -1,102 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - SUMMARYDIR = "${WORKSPACE}/summary" - REPORTDIR = "${WORKSPACE}/reports" - EARLYROOTDIR = 'early' - EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" - LATERROOTDIR = 'later' - LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" - - cleanfolders = [ - "${WORKSPACE}/build_artifacts", - "${EARLYDIR}", - "${LATERDIR}", - "${SUMMARYDIR}", - "${REPORTDIR}", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Obtain top level meta') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - - def earlyimportoptions = importer.getNewImportOptions() - earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) - earlyimportoptions.setincrementversionnumber(false) - earlyimportoptions.setfilter('meta/**') - importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) - - def laterimportoptions = importer.getNewImportOptions() - laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) - laterimportoptions.setincrementversionnumber(false) - laterimportoptions.setfilter('meta/**') - importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) - } - } - } - stage('Diff components') { - steps { - script { - def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") - (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,reports/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/documentation/doxygen_deploy.groovy b/Common_scripts/documentation/doxygen_deploy.groovy deleted file mode 100644 index 5054c02..0000000 --- a/Common_scripts/documentation/doxygen_deploy.groovy +++ /dev/null @@ -1,80 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Clean imports') { - steps { - script { - env.DOXYGEN = "doxygen" - cleanfolders = [ - DOXYGEN, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Import doxygen artifacts') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def importoptions = importer.getNewImportOptions() - - importoptions.jenkinsjob(BUILD_JOB_WITH_DOXYGEN_ARTIFACTS, BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR) - importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') - doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) - - productname = doxygenmeta.productinfo.productname - buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix - - echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}" - } - } - } - stage('Copy doxygen to server') { - steps { - script { - doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" - pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}" - echo "Copying from ${doxygensource} to ${pathonserver}" - if (isUnix()) { - sh """#!/bin/bash - rm -rfv "${pathonserver}" - if [ \$? -ne 0 ]; then exit 1; fi - - mkdir -pv "${pathonserver}" - if [ \$? -ne 0 ]; then exit 1; fi - - echo mv -v "${doxygensource}"/ "${pathonserver}" - mv -v "${doxygensource}"/ "${pathonserver}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } else { - error("This job does not run on Windows at the moment") - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/hardware/sominfo.groovy b/Common_scripts/hardware/sominfo.groovy deleted file mode 100644 index 72a3ca9..0000000 --- a/Common_scripts/hardware/sominfo.groovy +++ /dev/null @@ -1,80 +0,0 @@ -def sominfostore = null - -/* Example expected JSON -{ - "SomType": { - "Som_BA4": { - "Artifact info": { - "Location": "Jenkins job artifact", - "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", - "Job selector": "true", - "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", - "Release repo revision": "" - } - }, - "Som_FC5": { - "Artifact info": { - "Location": "Jenkins job artifact", - "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", - "Job selector": "true", - "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", - "Release repo revision": "" - } - } - } -} -*/ - -def String asJsonText() { - return writeJSON(returnText: true, json: sominfostore, pretty: 1) -} - -def boolean exists() { - return (sominfostore != null) -} - -def set(String jsontext) { - sominfostore = readJSON(text: jsontext) - echo "Som support info:\n${asJsonText()}" -} - -def get() { - return sominfostore -} - -def somtypes() { - return sominfostore['SomType'] -} - -def artifactinfo(somtype) { - return somtype['Artifact info'] -} - -def artifactlocation(artifactinfo) { - return artifactinfo['Location'] -} - -def boolean isjoblocation(artifactinfo) { - return (artifactinfo['Location'] == 'Jenkins job artifact' || - artifactinfo['Location'] == 'Jenkins job artefact') -} - -def boolean isreleaserepolocation(artifactinfo) { - return artifactinfo['Location'] == 'Release repository' -} - -def artifactjob(artifactinfo) { - return [ - "name": artifactinfo['Job'], - "selector": "${artifactinfo['Job selector']}" - ] -} - -def releaserepo(artifactinfo) { - return [ - "repo": artifactinfo['Release repo'], - "revision": artifactinfo['Release repo revision'] - ] -} - -return this \ No newline at end of file diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy deleted file mode 100644 index fa32ebf..0000000 --- a/Common_scripts/import/importer.groovy +++ /dev/null @@ -1,408 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - -class ImportOptions { - def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - - boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported - String m_filter // Only import files/paths/folders that match the filter - String m_excludes // Exclude files/paths/folders that match the exclude filter - boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts - boolean m_nested // Import all child artifacts too (identified in the related meta info - boolean m_flatten // Flatten files and folders imported - boolean m_includemetainfilter // Make sure meta is also included in filter - boolean m_clean // Clean import folder before obtaining artifacts - boolean m_appendtoparentmeta // Append to parent meta - String m_sourceselect // Where are the artifacts coming from - - String m_jobname - def m_buildnumberorselector - - String m_repo - String m_revision - - boolean m_showdetails - - ImportOptions(script, - String sourceselect = 'Jenkins job artifact', - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false, - boolean nested = false, - boolean flatten = false, - boolean includemetainfilter = true, - boolean clean = true, - appendtoparentmeta = true, - showdetails = false) { - m_script = script - m_sourceselect = sourceselect.trim() - m_incrementversionnumber = incrementversionnumber - m_filter = filter.trim() - m_excludes = excludes.trim() - m_fingerprint = fingerprint - m_nested = nested - m_flatten = flatten - m_includemetainfilter = includemetainfilter - m_clean = clean - m_appendtoparentmeta = appendtoparentmeta - m_showdetails = showdetails - } - - String show() { - return """m_incrementversionnumber: ${m_incrementversionnumber} - m_filter: ${m_filter} - m_excludes: ${m_excludes} - m_fingerprint: ${m_fingerprint} - m_nested: ${m_nested} - m_flatten: ${m_flatten} - m_includemetainfilter: ${m_includemetainfilter} - m_clean: ${m_clean} - m_appendtoparentmeta: ${m_appendtoparentmeta} - m_sourceselect: ${m_sourceselect} - m_jobname: ${m_jobname} - m_buildnumberorselector: ${m_buildnumberorselector} - m_repo: ${m_repo} - m_revision: ${m_revision} - """ - } - - void setsourceselect(String sourceselect) { - sourceselect = sourceselect.trim() - switch (sourceselect) { - case 'Jenkins job artefact': - m_sourceselect = 'Jenkins job artifact' - break - case 'Jenkins job artifact': - case 'Release repository': - m_sourceselect = sourceselect - break - default: - m_script.error("Unknown type of artifact source select: ${sourceselect}") - } - } - - void setincrementversionnumber(boolean incrementversionnumber = true) { - m_incrementversionnumber = incrementversionnumber - } - - void setfilter(String filter) { - m_filter = filter.trim() - } - - void setexcludes(String excludes) { - m_excludes = excludes.trim() - } - - void setfingerprint(boolean fingerprint = true) { - m_fingerprint = fingerprint - } - - void setnested(boolean nested = true) { - m_nested = nested - } - - void flattenimports(boolean flatten = true) { - m_flatten = flatten - } - - void includemetainfilter(boolean includemetainfilter = true) { - m_includemetainfilter = includemetainfilter - } - - void clean(boolean clean = true) { - m_clean = clean - } - - void setappendtoparentmeta(boolean appendtoparentmeta) { - m_appendtoparentmeta = appendtoparentmeta - } - - void jenkinsjob(String jobname, String buildnumberorselector) { - m_jobname = jobname.trim() - m_buildnumberorselector = [ - $class: 'ParameterizedBuildSelector', - parameterName: buildnumberorselector - ] - setsourceselect('Jenkins job artifact') - } - - void jenkinsjob(String jobname, int buildnumberorselector) { - m_jobname = jobname.trim() - m_buildnumberorselector = [ - $class: 'ParameterizedBuildSelector', - parameterName: "${buildnumberorselector}" - ] - setsourceselect('Jenkins job artifact') - } - - void releaserepo(String repo, String revision) { - m_repo = repo.trim() - m_revision = revision.trim() - setsourceselect('Release repository') - } - - void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) { - jenkinsjob(jobname, buildnumberorselector) - releaserepo(repo, revision) - setsourceselect(sourceselect) - } - - void showdetails(boolean showdetails) { - m_showdetails = showdetails - } -} - -ImportOptions getNewImportOptions() { - return new ImportOptions(this) -} - -def formFilter(ImportOptions importoptions) { - def String filter = importoptions.m_filter - if (filter != '') { - if (importoptions.m_includemetainfilter) { - filter = 'meta/**,' + filter - } - } - echo "Import options filter is: ${filter}" - return filter -} - -def conditionalClean(ImportOptions importoptions, String destpath) { - if (importoptions.m_clean) { - echo "Cleaning ${destpath}" - dir(destpath) { - deleteDir() - } - } -} - -def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_sourceselect != 'Jenkins job artifact') { - error("Import options settings have not been set for: Jenkins job artifact") - return - } - - conditionalClean(importoptions, destpath) - - echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" - echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" - copyArtifacts( - filter: formFilter(importoptions), - excludes: importoptions.m_excludes, - fingerprintArtifacts: importoptions.m_fingerprint, - projectName: importoptions.m_jobname, - selector: importoptions.m_buildnumberorselector, - flatten: importoptions.m_flatten, - target: destpath - ) - def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) - if (importoptions.m_incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) - } - return importedmeta -} - -def rewritemetafileasreleased(meta, String filepath) { - // Modify the meta info to say that is has already been released - meta.markreleased() - meta.saveJsonSerialised(filepath) -} - -def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_sourceselect != 'Release repository') { - error("Import options settings have not been set for: Release repository") - return - } - - def rooturl = importoptions.m_repo - def rootrevision = importoptions.m_revision - echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - - conditionalClean(importoptions, destpath) - - def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) - gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - - def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) - - echo "Getting associated metadata from path: ${destpath}" - def collectedmetalist = importedmeta.getassociated(destpath) - for(def collectedmeta in collectedmetalist) { - echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}" - } - - echo "Cloning associated repos inside path: ${destpath}" - for(def collectedmeta in collectedmetalist) { - def relatedmeta = collectedmeta.metainfo - def relatedoffsetpath = collectedmeta.accumulatedpath - def cloneurl = relatedmeta.releaseinfo.releaserepoURL - def clonerevision = "refs/tags/${relatedmeta.releasetag()}" - def clonepath = "${destpath}/${relatedoffsetpath}" - if (relatedmeta.componentidentifier() != '_' && - cloneurl != '') { - echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) - gh.stripgithistory(clonepath) - def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" - echo "Marking related meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(relatedmeta, metafilepath) - } else { - echo "Not importing component that would be placed at path: ${clonepath}" - } - } - gh.stripgithistory(destpath) - def metafilepath = "${destpath}/${meta.metafilepath()}" - echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(importedmeta, metafilepath) - - if (importoptions.m_incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) - } - return importedmeta -} - -def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_sourceselect != 'Release repository') { - error("Import options settings have not been set for: Release repository") - return - } - - conditionalClean(importoptions, destpath) - - def rooturl = importoptions.m_repo - def rootrevision = importoptions.m_revision - echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - - def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) - gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - - def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) - - gh.stripgithistory(destpath) - def metafilepath = "${destpath}/${meta.metafilepath()}" - echo "Marking meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(importedmeta, metafilepath) - - if (importoptions.m_incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) - } - return importedmeta -} - -def artifactimport(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_showdetails) { - echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" - } - switch (importoptions.m_sourceselect) { - case 'Jenkins job artifact': - case 'Jenkins job artefact': - return artifactimportfrombuildjob(meta, importoptions, destpath) - case 'Release repository': - if (importoptions.m_nested) { - return hierarchyartifactimportfromrepo(meta, importoptions, destpath) - } - return artifactimportfromrepo(meta, importoptions, destpath) - default: - error("Unknown type of artifact source: ${source}") - } - return meta -} - -// Deprecated (use above) -def artifactimport(meta, - String sourceselect, - String jenkinsjob, - buildnumberorselector, - String repo, - String revision, - String destpath, - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false, - boolean nested = false) { - echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" - def importoptions = importer.getNewImportOptions() - - importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - importoptions.setnested(nested) - - return artifactimport(meta, importoptions, destpath) -} - -// Deprecated (use above) -def artifactimport(meta, - String buildjob, - int buildnumber, - String destpath, - boolean incrementversionnumber = true, - String filter = '**', - String excludes = '', - boolean fingerprint = false) { - echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" - def importoptions = importer.getNewImportOptions() - - importoptions.jenkinsjob(buildjob, " ${buildnumber}") - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - - return artifactimport(meta, importoptions, destpath) -} - -// Deprecated (use above) -def artifactimport(meta, - String buildjob, - buildnumberorselector, - String destpath, - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false) { - echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" - def importoptions = importer.getNewImportOptions() - - importoptions.jenkinsjob(buildjob, buildnumberorselector) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - - return artifactimport(meta, importoptions, destpath) -} - -// Import the artifacts of a named child component found below parent meta's hierarchy -def artifactimport(meta, - parentmeta, - String parentmetapath, - String childproduct, - String childsubcomponent, - ImportOptions importoptions, - String destinationfolder) { - // Import from parent meta - def childmeta = meta.getNewMetaObject() - def childmeta2 - echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" - echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" - def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) - if (findresults.matchfound) { - childmeta = findresults.matchedmetaitem.metainfo - if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) { - importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) - } else { - importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) - } - childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) - } else { - error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") - } - return childmeta2 -} - -return this; diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy deleted file mode 100644 index 8dbf23e..0000000 --- a/Common_scripts/install/installhelper.groovy +++ /dev/null @@ -1,136 +0,0 @@ -import hudson.FilePath - -// Transfer file from intermediate position to final position; -// creating necessary folders on the way; -// deleting intermediate file -def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { - def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() - - def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" - def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" - def export_filepath = new hudson.FilePath(channel, abs_export_out) - def export_out_folder = export_filepath.getParent() - def export_out_folder_string = export_out_folder.toString() - def export_out_file = export_filepath.getName() - - echo "Creating folder ${export_out_folder_string}" - export_out_folder.mkdirs() - - echo "Copying from ${pathToIntermediate} to ${abs_export_out}" - def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) - intermediate_filepath.copyTo(export_filepath) - intermediate_filepath.delete() -} - -// Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { - artifactFetched = [:] - - if (showdetails) { - echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") - } - def importoptions = importer.getNewImportOptions() - importoptions.showdetails(showdetails) - for (installer in toBuild) { - def installerinfo = installerDetails.get(installer) - if (showdetails) { - echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") - } - def artifactKeys = installerinfo[1] - if (artifactKeys instanceof java.lang.String) { - artifactKeys = [ artifactKeys ] - } - if (showdetails) { - echo ("artifactKeys:\n${artifactKeys}") - } - for (artifactKey in artifactKeys) { - artifactDetail = artifactDetails.get(artifactKey) - if (showdetails) { - echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") - } - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail - if (artifactFetched.get(artifactKey) == null) { - importoptions.jenkinsjob(sourceProject, sourceselector) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(toCopy) - importoptions.setexcludes(notToCopy) - - def importmeta = importer.artifactimport(meta, importoptions, destFolder) - artifactFetched.put(artifactKey, importmeta) - } - } - } - if (showdetails) { - echo ("artifactFetched:\n${artifactFetched}") - } - return artifactFetched -} - -@NonCPS -def checkISMfilebackend(String xml) { - def String message = "" - - def parsedxml = new XmlParser(true, true, true).parseText(xml) - def pathvariableinfo = [:] - def exitcode = 0 - // Load up a map of path variable and path pairs - // Ignore blank path definitions - // Ignore the workspace definition which should have the only absolute path - parsedxml.each{ table -> - if (table.@name == "ISPathVariable") { - table['row'].each{ row -> - def pathname = row['td'][0].value()[0] - def path = row['td'][1].value()[0] - if (path && pathname != 'WORKSPACE') { - pathvariableinfo[pathname] = path - } - } - } - } - // Check that each path variable is defined beginning with a path variable or the WORKSPACE one - pathvariableinfo.each{ pair -> - message += "pair key: ${pair.key} value: ${pair.value}\n" - def path = pair.value - def pathvarmatcher = path =~ /<(.*)>.*/ - if (pathvarmatcher.matches()) { - def pathvarfound = pathvarmatcher.group(1) - if (pathvarfound != 'WORKSPACE') { - if (!pathvariableinfo.containsKey(pathvarfound)) { - message += "Undefined path variable ${pathvarfound} found\n" - exitcode = 1 - } - } - } else { - message += "Path: ${path} does not begin with a path variable\n" - exitcode = 1 - } - } - return [exitcode, message] -} - -def checkISMfile(String ISMfile) { - echo "Checking ISM file: ${ISMfile}" - def xml = readFile(ISMfile) - //echo "ISM file xml: ${xml}" - (exitcode, message) = checkISMfilebackend(xml) - echo message - if (exitcode > 0) { - error "ISM file errors found" - } else { - echo "Checked ISM file - ok" - } -} - -certificatethumbprints = [:] -certificatethumbprints['floreana'] = '0FDEB0A227982C877146CA595091173AC9655479' // Authoring takes place on this machine - so this is what should be in the ISM file already -certificatethumbprints['hawaii'] = '49FD1B361CFF35014B81940C5B046543045DD34B' - -def selectSigningCertificate(String nodeName, String ISMfile) { - def masterpattern = '0FDEB0A227982C877146CA595091173AC9655479' - echo "Substituting occurrences of ${masterpattern} with ${certificatethumbprints[nodeName]} in ISM file: ${ISMfile}" - - textutils_subst = load("${autobuilder.commonscripts()}/textutils/subst.groovy") - textutils_subst.substitute(ISMfile, masterpattern, certificatethumbprints[nodeName]) -} - -return this; diff --git a/Common_scripts/meta/folder.groovy b/Common_scripts/meta/folder.groovy deleted file mode 100644 index 6715145..0000000 --- a/Common_scripts/meta/folder.groovy +++ /dev/null @@ -1,1193 +0,0 @@ -import java.nio.file.Paths -import java.nio.file.Path -import groovy.json.JsonSlurperClassic - -abstract class GISJsonSerialised implements Serializable { - def script - - GISJsonSerialised(script) { - this.script = script - } - - abstract LinkedHashMap serialisationdata() - - abstract void deserialiseJson(rawmeta) - - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) - } -} - -class ProductInfo extends GISJsonSerialised { - private String format - private String productname - private String subcomponentname - private String variant - private String uniqueid - - ProductInfo(script, String productname = '', String subcomponentname = '', String variant = '') { - super(script) - this.format = '0.1' - this.productname = productname - this.subcomponentname = subcomponentname - this.variant = variant - this.uniqueid = UUID.randomUUID().toString() - } - - void set(String productname, String subcomponentname, String variant = '') { - // Make sure the fields can be used in git tags - def int pstatus = 0 - if (script.isUnix()) { - pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) - } - else { - pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) - } - if (pstatus > 0) { - script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") - } - def int sstatus = 0 - if (script.isUnix()) { - sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) - } - else { - sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) - } - if (sstatus > 0) { - script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") - } - if (variant) { - def int vstatus = 0 - if (script.isUnix()) { - vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) - } - else { - vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) - } - if (vstatus > 0) { - script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") - } - } - this.productname = productname - this.subcomponentname = subcomponentname - this.variant = variant - } - - void set(ProductInfo otherinfo) { - this.productname = otherinfo.productname - this.subcomponentname = otherinfo.subcomponentname - this.variant = otherinfo.variant - this.uniqueid = otherinfo.uniqueid - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - productname = rawmeta['productname'] - subcomponentname = rawmeta['subcomponentname'] - variant = rawmeta['variant'] ?: '' - uniqueid = rawmeta['uniqueid'] ?: uniqueid - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'productname' : productname, - 'subcomponentname' : subcomponentname, - 'variant' : variant, - 'uniqueid': uniqueid, - ] - return data - } -} - -class GitInfo extends GISJsonSerialised { - private String format - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo - - GitInfo(script, sourcepath = '', sourcerevision = '', gitrevision = '', remoteurl = '', remoteinfo = '') { - super(script) - this.format = '0.1' - this.sourcepath = sourcepath - this.sourcerevision = sourcerevision - this.gitrevision = gitrevision - this.remoteurl = remoteurl - this.remoteinfo = remoteinfo - } - - GitInfo(GitInfo another) { - super(another.script) - this.sourcepath = another.sourcepath - this.sourcerevision = another.sourcerevision - this.gitrevision = another.gitrevision - this.remoteurl = another.remoteurl - this.remoteinfo = another.remoteinfo - } - - void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { - this.sourcepath = sourcepath - this.sourcerevision = sourcerevision - this.gitrevision = gitrevision - this.remoteurl = remoteurl - this.remoteinfo = remoteinfo - } - - void set(GitInfo otherinfo) { - this.sourcepath = otherinfo.sourcepath - this.sourcerevision = otherinfo.sourcerevision - this.gitrevision = otherinfo.gitrevision - this.remoteurl = otherinfo.remoteurl - this.remoteinfo = otherinfo.remoteinfo - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - sourcepath = rawmeta['sourcepath'] - sourcerevision = rawmeta['sourcerevision'] - gitrevision = rawmeta['gitrevision'] - remoteurl = rawmeta['remoteurl'] - remoteinfo = rawmeta['remoteinfo'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'sourcepath' : sourcepath, - 'sourcerevision' : sourcerevision, - 'gitrevision' : gitrevision, - 'remoteurl' : remoteurl, - 'remoteinfo' : remoteinfo, - ] - return data - } -} - -class JenkinsBuildParameter extends GISJsonSerialised { - private String name - private String valuetype - private String value - - JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { - super(script) - this.name = name - this.valuetype = valuetype - this.value = value - } - - void set(String name, String valuetype, String value) { - this.name = name - this.valuetype = valuetype - this.value = value - } - - void deserialiseJson(rawmeta) { - name = rawmeta['name'] - valuetype = rawmeta['valuetype'] - value = rawmeta['value'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'name' : name, - 'valuetype' : valuetype, - 'value' : value, - ] - return data - } -} - -class JenkinsBuildInfo extends GISJsonSerialised { - private String format - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate - private ArrayList parameters - private boolean isfullbuild - - JenkinsBuildInfo(script, String buildjobname = '', int buildnumber = 0, String buildmachine = '', String builddate = '', boolean isfullbuild = true) { - super(script) - this.format = '0.1' - this.buildjobname = buildjobname - this.buildnumber = buildnumber - this.buildmachine = buildmachine - this.builddate = builddate - this.isfullbuild = isfullbuild - parameters = new ArrayList() - } - - void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { - this.buildjobname = buildjobname - this.buildnumber = buildnumber - this.buildmachine = buildmachine - this.builddate = builddate - this.isfullbuild = isfullbuild - } - - void set(boolean isfullbuild) { - this.isfullbuild = isfullbuild - } - - boolean isFullBuild() { - return isfullbuild - } - - void addParameter(String name, value) { - def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) - parameters.add(p) - return p - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - buildjobname = rawmeta['buildjobname'] - buildnumber = rawmeta['buildnumber'] - buildmachine = rawmeta['buildmachine'] - builddate = rawmeta['builddate'] - isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] - parameters = new ArrayList() - if (rawmeta['isfullbuild'] != null) { - for (rawparameter in rawmeta['parameters']) { - def p = new JenkinsBuildParameter(script) - p.deserialiseJson(rawparameter) - parameters.add(p) - } - } - } - - List parametersserialised() { - def associated = [] - for (p in parameters) { - associated.add(p.serialisationdata()) - } - return associated - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'buildjobname' : buildjobname, - 'buildnumber' : buildnumber, - 'buildmachine' : buildmachine, - 'builddate' : builddate, - 'isfullbuild' : isfullbuild, - 'parameters' : parametersserialised(), - ] - return data - } -} - -class BuildStampInfo extends GISJsonSerialised { - private String format - private String buildstampprefix - private int revision - private String buildstamp - - BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { - super(script) - this.format = '0.1' - this.buildstampprefix = buildstampprefix - this.revision = revision - this.buildstamp = buildstamp - } - - BuildStampInfo(BuildStampInfo another) { - super(another.script) - this.buildstampprefix = another.buildstampprefix - this.revision = another.revision - this.buildstamp = another.buildstamp - } - - void set(String buildstampprefix, int revision, String buildstamp) { - this.buildstampprefix = buildstampprefix - this.revision = revision - this.buildstamp = buildstamp - } - - void set(BuildStampInfo other) { - set(other.buildstampprefix, other.revision, other.buildstamp) - } - - // Override the buildstamp - void override(String buildstamp) { - this.buildstamp = buildstamp - script.echo("Build stamp overridden to: ${this.buildstamp}") - def s = this.buildstamp.split('\\.') - if (s.size() != 4) { - script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") - } - this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" - script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") - this.revision = s[3].toInteger() - script.echo("Revision overridden to: ${this.revision}") - } - - String buildstampcommasep() { - def String buildstampcommasep = buildstamp.replace(".", ",") - return buildstampcommasep - } - - void changerevision(int newrevision) { - script.echo("Build stamp currently ${buildstamp}") - revision = newrevision - def List buildstampsplit = buildstamp.split('\\.') - buildstampsplit[-1] = newrevision.toString() - buildstamp = buildstampsplit.join('.') - script.echo("Build stamp now ${buildstamp}") - } - - void set(String buildstampprefix, int revision) { - buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' - this.buildstampprefix = buildstampprefix - this.buildstamp = buildstampprefix + "0" - changerevision(revision) - } - - void set(String buildstampprefix, String revision = '0') { - set(buildstampprefix, revision.toInteger()) - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - buildstampprefix = rawmeta['buildstampprefix'] - revision = rawmeta['revision'] - buildstamp = rawmeta['buildstamp'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'buildstampprefix' : buildstampprefix, - 'revision' : revision, - 'buildstamp' : buildstamp, - ] - return data - } -} - -class ReleaseInfo extends GISJsonSerialised { - private String format - private String releaserepoURL - private ArrayList releaseitems - private boolean released - - ReleaseInfo(script, String releaserepoURL = '', boolean released = false) { - super(script) - this.format = '0.1' - this.releaserepoURL = releaserepoURL - this.releaseitems = new ArrayList() - this.released = released - } - - void set(String releaserepoURL = '') { - this.releaserepoURL = releaserepoURL - } - - void addReleaseItem(String releaseitem) { - script.echo("Adding release item: ${releaseitem} to release items") - releaseitems.add(releaseitem) - } - - void markreleased() { - this.released = true - } - - List releaseitemssserialised() { - def items = [] - for (ri in releaseitems) { - items.add(ri) - } - return items - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - releaserepoURL = rawmeta['releaserepoURL'] - releaseitems = new ArrayList() - for (rawreleaseitem in rawmeta['releaseitems']) { - releaseitems.add(rawreleaseitem) - } - released = rawmeta['released'] ?: false - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'releaserepoURL' : releaserepoURL, - 'releaseitems' : releaseitemssserialised(), - 'released' : released - ] - return data - } -} - -class ConfigurationInfo extends GISJsonSerialised { - private String configurationname - private String configurationURL - private String configurationversion - private String configurationSHA - - ConfigurationInfo(script, configurationname = '', configurationURL = '', configurationversion = '', configurationSHA = '') { - super(script) - this.configurationname = configurationname - this.configurationURL = configurationURL - this.configurationversion = configurationversion - this.configurationSHA = configurationSHA - } - - void deserialiseJson(rawmeta) { - configurationname = rawmeta['configurationname'] - configurationURL = rawmeta['configurationURL'] - configurationversion = rawmeta['configurationversion'] - configurationSHA = rawmeta['configurationSHA'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'configurationname' : configurationname, - 'configurationURL' : configurationURL, - 'configurationversion' : configurationversion, - 'configurationSHA' : configurationSHA, - ] - return data - } -} - -class ChecksumPair extends GISJsonSerialised { - private String filepath - private String sum - - ChecksumPair(script, String filepath = '', String sum = '') { - super(script) - this.filepath = filepath - this.sum = sum - } - - void deserialiseJson(rawmeta) { - filepath = rawmeta['filepath'] - sum = rawmeta['sum'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'filepath' : filepath, - 'sum' : sum, - ] - return data - } -} - -class ChecksumSet extends GISJsonSerialised { - private String format - private String artifactpath - private ArrayList checksumpairs - - ChecksumSet(script, artifactpath = '') { - super(script) - this.format = '0.1' - this.artifactpath = artifactpath - checksumpairs = new ArrayList() - } - - ChecksumPair addChecksumPair(String filepath, String sum) { - def cp = new ChecksumPair(script, filepath, sum) - checksumpairs.add(cp) - return cp - } - - List checksumpairsserialised() { - def associated = [] - for (cp in checksumpairs) { - associated.add(cp.serialisationdata()) - } - return associated - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - artifactpath = rawmeta['artifactpath'] - checksumpairs = new ArrayList() - for (rawchecksumpair in rawmeta['checksumpairs']) { - def csp = new ChecksumPair(script) - csp.deserialiseJson(rawchecksumpair) - checksumpairs.add(csp) - } - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'artifactpath' : artifactpath, - 'checksumpairs' : checksumpairsserialised(), - ] - return data - } -} - -class ChecksumInfo extends GISJsonSerialised { - private ArrayList checksumsets - - ChecksumInfo(script) { - super(script) - checksumsets = new ArrayList() - } - - ChecksumSet addChecksumSet(String artifactpath) { - def cs = new ChecksumSet(script, artifactpath) - checksumsets.add(cs) - return cs - } - - List checksumsetsserialised() { - def associated = [] - for (cs in checksumsets) { - associated.add(cs.serialisationdata()) - } - return associated - } - - void deserialiseJson(rawmeta) { - checksumsets = new ArrayList() - for (rawchecksumset in rawmeta['checksumsets']) { - def cs = new ChecksumSet(script) - cs.deserialiseJson(rawchecksumset) - checksumsets.add(cs) - } - } - - LinkedHashMap serialisationdata() { - def data = [ - 'checksumsets' : checksumsetsserialised(), - ] - return data - } -} - -class PublishableInfo extends GISJsonSerialised { - private String format - private String filepath - - PublishableInfo(script, String filepath = '') { - super(script) - this.format = '0.1' - this.filepath = filepath - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - filepath = rawmeta['filepath'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'filepath' : filepath, - ] - return data - } -} - -class MetaInfo extends GISJsonSerialised { - private static String METAFOLDERNAME = 'meta' - private static String METAFILENAME = 'info.json' - private static String METAARTIFACTS = "${METAFOLDERNAME}/**" - private static String format = '0.2' - - abstract class RelatedMetaInfo extends GISJsonSerialised { - String format - - RelatedMetaInfo(script, format) { - super(script) - this.format = format - } - - abstract void deserialiseJson(rawmeta) - abstract LinkedHashMap serialisationdata() - abstract String getrelatedid() - abstract void markreleased() - } - - class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { - private MetaInfo meta - private String offsetpath - private String uniqueid - - RelatedMetaInfo_v0_1(script, String offsetpath = '', MetaInfo meta = null, String uniqueid = UUID.randomUUID().toString()) { - super(script, '0.1') - this.offsetpath = offsetpath - this.uniqueid = uniqueid - this.meta = meta - } - - void deserialiseJson(rawmeta) { - offsetpath = rawmeta['offsetpath'] - uniqueid = rawmeta['uniqueid'] ?: uniqueid - meta = new MetaInfo(script) - meta.deserialiseJson(rawmeta['meta']) - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'offsetpath' : offsetpath, - 'uniqueid' : uniqueid, - 'meta' : meta.serialisationdata(), - ] - return data - } - - String getrelatedid() { - return uniqueid - } - - void markreleased() { - meta.releaseinfo.markreleased() - } - } - - class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { - private String importpath - private String offsetpath - private String uniqueid - - RelatedMetaInfo_v0_2(script, String importpath = '', String offsetpath = '', String uniqueid = UUID.randomUUID().toString()) { - super(script, '0.2') - this.importpath = importpath - this.offsetpath = offsetpath - this.uniqueid = uniqueid - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - offsetpath = rawmeta['offsetpath'] - uniqueid = rawmeta['uniqueid'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'offsetpath' : offsetpath, - 'uniqueid' : uniqueid - ] - return data - } - - String getrelatedid() { - return uniqueid - } - - void markreleased() { - // Do nothing - } - } - - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo - private ChecksumInfo checksuminfo - private ArrayList configurationinfo - private ArrayList publishableinfo - private LinkedHashMap custominfo - private ArrayList associatedmeta - - MetaInfo(script) { - super(script) - productinfo = new ProductInfo(script) - gitinfo = new GitInfo(script) - jenkinsbuildinfo = new JenkinsBuildInfo(script) - buildstampinfo = new BuildStampInfo(script) - releaseinfo = new ReleaseInfo(script) - checksuminfo = new ChecksumInfo(script) - configurationinfo = new ArrayList() - publishableinfo = new ArrayList() - custominfo = new LinkedHashMap() - associatedmeta = new ArrayList() - } - - void markreleased() { - script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") - releaseinfo.markreleased() - for (associated in associatedmeta) { - associated.markreleased() - } - } - - List configurationinfoserialised() { - def configuration = [] - for (configurationitem in configurationinfo) { - configuration.add(configurationitem.serialisationdata()) - } - return configuration - } - - void addPublishable(String filepath) { - def PublishableInfo pi = new PublishableInfo(script, filepath) - script.echo("Adding ${filepath} to publishable list") - publishableinfo.add(pi) - } - - void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { - def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) - script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") - configurationinfo.add(ci) - return ci - } - - void addCustomInfo(String customId, String customValue) { - custominfo.put(customId.trim(), customValue) - } - - String getCustomInfo(String customId) { - return custominfo.getAt(customId.trim()) - } - - def isFullBuild() { - return jenkinsbuildinfo.isFullBuild() - } - - List publishableinfoserialised() { - def publishable = [] - for (publishableitem in publishableinfo) { - publishable.add(publishableitem.serialisationdata()) - } - return publishable - } - - List associatedmetaserialised() { - def associated = [] - for (metaitem in associatedmeta) { - associated.add(metaitem.serialisationdata()) - } - return associated - } - - void deserialiseJson(rawmeta) { - if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { - script.error("Unsupported meta format found") - return - } - productinfo.deserialiseJson(rawmeta['productinfo']) - gitinfo.deserialiseJson(rawmeta['gitinfo']) - jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) - buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) - releaseinfo.deserialiseJson(rawmeta['releaseinfo']) - checksuminfo.deserialiseJson(rawmeta['checksuminfo']) - for (rawconfig in rawmeta['configuration']) { - def cf = new ConfigurationInfo(script) - cf.deserialiseJson(rawconfig) - configurationinfo.add(cf) - } - for (rawpublishable in rawmeta['publishableinfo']) { - def pb = new PublishableInfo(script) - pb.deserialiseJson(rawpublishable) - publishableinfo.add(pb) - } - custominfo = rawmeta['custominfo'] - for (rawrelatedmeta in rawmeta['associatedmeta']) { - if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { - def rm = new RelatedMetaInfo_v0_1(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) - } else if (rawrelatedmeta['format'] == '0.2') { - def rm = new RelatedMetaInfo_v0_2(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) - } - } - } - - def deserialiseJson(String filepath) { - script.echo("Deserialising JSON at file: ${filepath}") - def rawmeta = script.readJSON(file: filepath) - deserialiseJson(rawmeta) - } - - def deserialiseJsonSlurper(String filepath) { - File jsonfile = new File(filepath) - def jsonfiletext = jsonfile.getText() - def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) - deserialiseJson(rawmeta) - } - - String relativize(String mypath) { - String pathRelative - script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") - if (mypath.startsWith(script.WORKSPACE)) { - pathRelative = mypath.minus(script.WORKSPACE) - script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") - } else { - pathRelative = mypath - script.echo("Path relative is unchanged: ${pathRelative}") - } - // Convert \ with / - pathRelative = pathRelative.tokenize('\\').join('/') - // Strip leading / if any - pathRelative = pathRelative.replaceAll('^/+', '') - - script.echo("Path relative is ${pathRelative}") - - return pathRelative - } - - static String metafilepath() { - return METAFOLDERNAME + "/" + METAFILENAME - } - - MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { - MetaInfo mi = new MetaInfo(script) - - def String adjustedpath = relativize(destpath) - def importedmetafile = adjustedpath + "/" + metafilepath() - if (script.fileExists(importedmetafile)) { - script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") - mi.deserialiseJson(importedmetafile) - - RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) - if (appendtoparentmeta) { - associatedmeta.add(rmi) - } - } else { - script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") - addConfigurationInfo('', givenRepo, givenRevision, givenSHA) - } - return mi - } - - def copyrelatedmeta() { - script.echo("Copying related input meta data to output meta") - for (relatedmetainfo in associatedmeta) { - def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" - def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" - script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") - script.fileOperations([ - script.folderCreateOperation(folderPath: destpath), - script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) - ]) - } - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'productinfo' : productinfo.serialisationdata(), - 'gitinfo' : gitinfo.serialisationdata(), - 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), - 'buildstampinfo' : buildstampinfo.serialisationdata(), - 'releaseinfo' : releaseinfo.serialisationdata(), - 'configuration' : configurationinfoserialised(), - 'checksuminfo' : checksuminfo.serialisationdata(), - 'publishableinfo' : publishableinfoserialised(), - 'custominfo' : custominfo, - 'associatedmeta' : associatedmetaserialised(), - ] - return data - } - - static String folder() { - return METAFOLDERNAME - } - - def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here - ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here - String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" - ) { - def String artifactlist = METAARTIFACTS - releaseinfo.addReleaseItem(METAARTIFACTS) - for (releaseitem in releasedfromhere) { - artifactlist += "," + releaseitem - releaseinfo.addReleaseItem(releaseitem) - } - for (releaseitem in releasedfromsubmeta) { - artifactlist += "," + releaseitem - } - saveJsonSerialised(metafilepath) - return artifactlist - } - - String componentidentifier() { - return productinfo.productname + "_" + productinfo.subcomponentname - } - - String componentversionuniqueidentifier() { - return componentidentifier() + "_" + buildstampinfo.buildstamp - } - - String globalreleasetag() { - return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber - } - - String releasetag() { - return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber - } - - String releasetagfordebpackage() { - return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber - } - - class CollectedMeta { - private MetaInfo metainfo - private String accumulatedpath - private String accumulatedmetapath - - CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { - this.metainfo = metainfo - this.accumulatedpath = accumulatedpath - this.accumulatedmetapath = accumulatedmetapath - } - } - - class IterationInfo { - private ArrayList metacollection - private Map metamap - private String comment - private int level - private String gatheredpath - private String gatheredmetapath - private boolean limiteddepth - private int maxdepth - - IterationInfo(ArrayList metacollection, - Map metamap, - String comment = '', - int level = 0, - String gatheredpath = '', - String gatheredmetapath = '', - boolean limiteddepth = false, - int maxdepth = 0) { - this.metacollection = metacollection - this.metamap = metamap - this.comment = comment - this.level = level - this.gatheredpath = gatheredpath - this.gatheredmetapath = gatheredmetapath - this.limiteddepth = limiteddepth - this.maxdepth = maxdepth - } - - String show() { - return "Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}" - } - - void add(CollectedMeta item) { - metacollection.add(item) - metamap.put(item.metainfo.productinfo.uniqueid, item) - } - } - - def openchildmeta(IterationInfo ii, RelatedMetaInfo relatedmetainfo) { - def MetaInfo mi = new MetaInfo(script) - - def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") - def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" - def boolean imported - // If this has already been read in from a file - if (relatedmetainfo.getrelatedid() in ii.metamap) { - //script.echo("Already read in meta associated with uniqueid: ${relatedmetainfo.getrelatedid()} - Using it") - mi = ii.metamap[relatedmetainfo.getrelatedid()].metainfo - imported = true - } else if (script.fileExists(importedmetafile)) { - mi.deserialiseJson(importedmetafile) - imported = true - } else { - script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") - imported = false - } - - def childmetainfo = [ - 'imported': imported, - 'nextgatheredpath': nextgatheredpath, - 'nextgatheredmetapath': nextgatheredmetapath, - 'childmeta': mi - ] - return childmetainfo - } - - def walkrelated_v0_1(IterationInfo ii) { - //script.echo("Walk related v0.1: ${ii.show()}") - if (!ii.limiteddepth || ii.maxdepth > 0) { - ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) - ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { - def MetaInfo othermeta = relatedmetainfo.meta - def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") - def nextii = new IterationInfo(ii.metacollection, - ii.metamap, - ii.comment, - ii.level + 1, - nextgatheredpath, - nextgatheredmetapath, - ii.limiteddepth, - ii.maxdepth - 1) - def results = othermeta.walkrelated_v0_1(nextii) - ii.metacollection = results.metacollection - ii.metamap = results.metamap - ii.comment = results.comment - } else { - script.error("There is non v0.1 format related meta info under v0.1 related meta info") - } - } - } - return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] - } - - def walkrelated(IterationInfo ii) { - //script.echo("Walk related: ${ii.show()}") - if (!ii.limiteddepth || ii.maxdepth > 0) { - ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) - ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - //script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") - if (relatedmetainfo.format == '0.2') { - def childmetainfo = openchildmeta(ii, relatedmetainfo) - if (childmetainfo.imported) { - def nextii = new IterationInfo(ii.metacollection, - ii.metamap, - ii.comment, - ii.level + 1, - childmetainfo.nextgatheredpath, - childmetainfo.nextgatheredmetapath, - ii.limiteddepth, - ii.maxdepth - 1) - def results = childmetainfo.childmeta.walkrelated(nextii) - ii.metacollection = results.metacollection - ii.metamap = results.metamap - ii.comment = results.comment - } - } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { - def MetaInfo othermeta = relatedmetainfo.meta - def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") - def nextii = new IterationInfo(ii.metacollection, - ii.metamap, - ii.comment, - ii.level + 1, - nextgatheredpath, - nextgatheredmetapath, - ii.limiteddepth, - ii.maxdepth - 1) - def results = othermeta.walkrelated_v0_1(nextii) - ii.metacollection = results.metacollection - ii.metamap = results.metamap - ii.comment = results.comment - } - } - } - return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] - } - - // Walks all layers of associated data only - def allrelatedmeta(String metapath = '') { - def ArrayList metacollection = new ArrayList() - def Map metamap = new LinkedHashMap() - return walkrelated(new IterationInfo(metacollection, metamap, '', 0, '', metapath)) - } - - ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = new ArrayList() - def Map metamap = new LinkedHashMap() - def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) - script.echo("Release walker iteration info at start:\n${ii.show()}") - def results = walkrelated(ii) - return results.metacollection - } - - // Walks 1 layer of associated data only - def getassociated(String rootpath) { - return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself) - } - - Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { - if (releasemetacollection.containsKey(uniquestring)) { - def existingentry = releasemetacollection[uniquestring] - def existingentrypathlength = existingentry.accumulatedpath.split('/').size() - def newentrypathlength = metaitem.accumulatedpath.split('/').size() - if (existingentrypathlength > newentrypathlength) { - releasemetacollection.put(uniquestring, metaitem) - } - } else { - releasemetacollection.put(uniquestring, metaitem) - } - return releasemetacollection - } - - Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - - def Map versionuniquecollection = new LinkedHashMap() - for (metaitem in metacollection) { - def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() - versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) - } - - return versionuniquecollection - } - - Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - - def Map uniquecollection = new LinkedHashMap() - for (metaitem in metacollection) { - def String uniquestring = metaitem.metainfo.componentidentifier() - uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) - } - - return uniquecollection - } - - MetaInfo getNewMetaObject(script) { - return new MetaInfo(script) - } - - def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { - def results = [ - 'matchedmetaitem' : new CollectedMeta(this), - 'matchfound' : false - ] - def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") - for (dependentmetaitem in walkresults.metacollection) { - script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") - if (dependentmetaitem.metainfo.productinfo.productname == productname && - dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { - script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") - results.matchedmetaitem = dependentmetaitem - results.matchfound = true - break - } - } - return results - } -} - -MetaInfo metareset(script) { - def mi = new MetaInfo(script) - - echo "Resetting meta folder" - dir(mi.folder()) { - deleteDir() - } - - return mi -} - -MetaInfo metafromfolder(String folder = '') { - MetaInfo buildmeta = new MetaInfo(this) - buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") - return buildmeta -} - -MetaInfo metafromfolderSlurper(String folder = '') { - MetaInfo buildmeta = new MetaInfo(this) - buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") - return buildmeta -} - -return this; diff --git a/Common_scripts/packaging/debian.groovy b/Common_scripts/packaging/debian.groovy deleted file mode 100644 index 9f2f4d3..0000000 --- a/Common_scripts/packaging/debian.groovy +++ /dev/null @@ -1,111 +0,0 @@ -class DebRootDefinition { - String ROOTPATH - String PACKAGE_PREFIX - String PACKAGE_FOLDER_NAME - String PACKAGE_NAME - String UNIQUE_PACKAGE_NAME - - String PACKAGE_SOURCE_ROOT - String PACKAGE_SOURCE_FOLDER_NAME - String DEBPATH - - String ETC - String GIS_OPT_BIN - String GIS_OPT_LIB - String GIS_OPT_SCRIPTS_APP - String GIS_OPT_SCRIPTS_EL - String GIS_OPT_SCRIPTS_CONTROL - String GIS_OPT_WWW - String GIS_OPT_OTHER - String GIS_OPT_KERNEL - String GIS_OPT_LOGIC - String GIS_OPT_PRINTHEAD - String GIS_OPT_BOOT - - String USR_BIN - String USR_LIB_ARCH - String LIB_FIRMWARE - String HTTP_DEV_DIV - String LIB_SYSTEMD - - DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix) { - ROOTPATH = rootpath - PACKAGE_PREFIX = packageprefix - PACKAGE_FOLDER_NAME = packageprefix - PACKAGE_NAME = packageprefix + "_armhf.deb" - UNIQUE_PACKAGE_NAME = packageprefix + "_" + buildstamp + "_armhf.deb" - - PACKAGE_SOURCE_ROOT = packagesourceroot - PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" - DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" - - ETC = "${DEBPATH}/etc" + inpackagepathsuffix - GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + inpackagepathsuffix - GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + inpackagepathsuffix - GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + inpackagepathsuffix - GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + inpackagepathsuffix - GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" + inpackagepathsuffix - GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + inpackagepathsuffix - GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + inpackagepathsuffix - GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" + inpackagepathsuffix - GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + inpackagepathsuffix - GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + inpackagepathsuffix - GIS_OPT_BOOT = "${DEBPATH}/opt/gis/boot" + inpackagepathsuffix - - USR_BIN = "${DEBPATH}/usr/bin" + inpackagepathsuffix - USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + inpackagepathsuffix - LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + inpackagepathsuffix - HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + inpackagepathsuffix - LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + inpackagepathsuffix - } -} - -def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix = '') { - return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) -} - -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { - withCredentials([usernamePassword(credentialsId: credentials_id, - usernameVariable: 'SUDO_USERNAME', - passwordVariable: 'SUDO_PASSWORD')]) { - def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" - def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" - try { - sh """#!/bin/bash - # Exit if any command fails - set -xe - - cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - - ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" - - mkdir -v -p "${packageout}" - echo "Building package:" - dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" - - mkdir -v -p "${uniquepackageout}" - cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" - - echo "Showing package:" - dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package contents:" - dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - - echo "Showing package lintian output:" - lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - - ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - """ - } - catch (Exception e) { - echo "Exception occurred" - sh """#!/bin/bash -v - ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - - exit 1 - """ - } - } -} - -return this; \ No newline at end of file diff --git a/Common_scripts/platform/lineendings.groovy b/Common_scripts/platform/lineendings.groovy deleted file mode 100644 index 9657b0d..0000000 --- a/Common_scripts/platform/lineendings.groovy +++ /dev/null @@ -1,5 +0,0 @@ - -// Put ${nl} into strings when wanting a newline -nl = System.properties['line.separator'] - -return this; diff --git a/Common_scripts/publish/.classpath b/Common_scripts/publish/.classpath deleted file mode 100644 index 287c1ab..0000000 --- a/Common_scripts/publish/.classpath +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/Common_scripts/publish/.gitignore b/Common_scripts/publish/.gitignore deleted file mode 100644 index abed335..0000000 --- a/Common_scripts/publish/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/do$_publish_closure4.class -/do$_resetfolders_closure1.class -/do$_resetfolders_closure2.class -/do$_resetfolders_closure3.class -/do$_stripgithistory_closure5.class -/do.class -/publish_item_select.class diff --git a/Common_scripts/publish/.project b/Common_scripts/publish/.project deleted file mode 100644 index 5c60328..0000000 --- a/Common_scripts/publish/.project +++ /dev/null @@ -1,18 +0,0 @@ - - - publish_item_select - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.groovy.core.groovyNature - org.eclipse.jdt.core.javanature - - diff --git a/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs deleted file mode 100644 index 1e68ed6..0000000 --- a/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs +++ /dev/null @@ -1,3 +0,0 @@ -eclipse.preferences.version=1 -groovy.compiler.level=25 -groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/Common_scripts/publish/do.groovy b/Common_scripts/publish/do.groovy deleted file mode 100644 index ea090f0..0000000 --- a/Common_scripts/publish/do.groovy +++ /dev/null @@ -1,174 +0,0 @@ -import groovy.io.FileType - -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" -releaserepofolder = "${WORKSPACE}/releaserepos" -filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(filteredreleaserepofolder) { - deleteDir() - } -} - -def productuniquefolder(releasemeta) { - return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" -} - -def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { - return "Software/${productline} ${softwarerelease}/${productuniquefolder(releasemeta)}" -} - -def uniqueabsoluterreleasefolder(String productline, String releaserepofolder, String softwarerelease, releasemeta) { - return "${releaserepofolder}/${uniquereleasefolder(productline, softwarerelease, releasemeta)}" -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Publish summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason = reason -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def guessreporootsuffix(componentURL) { - def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - return guess -} - -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { - summaryappend("Artifacts: ") - - def releasemeta = releaseinfo.metainfo - def String componentURL = releasemeta.releaseinfo.releaserepoURL - def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" - def publishableitems = releasemeta.publishableinfo - - def ArrayList publishablepaths = new ArrayList() - if (publishableitems.size() > 0) { - for (publishableitem in publishableitems) { - def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" - echo "publishableitem: ${productuniquepublishableitem}" - if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { - publishablepaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") - } - } - } - if (publishablepaths.size() > 0) { - def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone - } - else { - summaryappend("None ") - } -} - -def publish(hostedftpsiteinfo, String customer, String finalreleaserepofolder = releaserepofolder) { - summaryappend(" FTP operations: ") - - def String ftproot = "${hostedftpsiteinfo.workingdirectory}${customer}" - dir(finalreleaserepofolder) { - summaryappend("FTP operations: Transferring to ${ftproot}") - sh """#!/bin/bash - ncftpput -u ${hostedftpsiteinfo.username} -p ${hostedftpsiteinfo.password} -R ${hostedftpsiteinfo.url} "${ftproot}" . - """ - summaryappend(", Transferred ") - } -} - -def copydirectory(File dirFrom, File dirTo) { - if (!dirTo.exists()) { - dirTo.mkdirs(); - } - dirFrom.eachFile(FileType.FILES) { File source -> - File target = new File(dirTo, source.getName()) - target.bytes = source.bytes - } - dirFrom.eachFile(FileType.DIRECTORIES) { File source -> - File target = new File(dirTo, source.getName()) - copyLargeDir(source, target) - } -} - -def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { - def releasemeta = releaseinfo.metainfo - def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) - - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - - getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) - - summaryappend("\n") -} - -def convertitemstopublish(String itemstopublishstring) { - def List paths = [] - - def String[] items = itemstopublishstring.split(',') - def String allpaths = "" - for(String item : items) { - def String[] pathitems = item.split('#') - def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" - paths.add(path) - allpaths += "${path}\n" - } - echo "Selected item to publish:\n${allpaths}" - return paths -} - -// Publish selected items from a release to the customer folder on the ftp site -// itemstopublishstring is a comma-seperated set of strings in this format ## -def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { - int md = maxdepth.toInteger() - resetfolders() - preconditions() - - def productline = releasemeta.productinfo.productname - // Walk the meta information to a max depth, looking for artifacts to publish - def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) - - echo "Version unique components:" - def String versionuniquecomponent = "" - for (key in componentReleaseInfoVersionUnique.keySet()) { - versionuniquecomponent += "${key}\n" - } - echo versionuniquecomponent - - def itempathstopublish = convertitemstopublish(itemstopublishstring) - for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - recallcomponent(productline, releaseinfo, softwarerelease, itempathstopublish) - } - gh.stripgithistory(releaserepofolder) - publish(hostedftpsiteinfo, customer) - - return [summarystring, markunstable, unstablereason] -} - -def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String customer, String softwarerelease, String maxdepth) { - return publishhierarchy(hostedftpsiteinfo, releasemeta, releasemetapath, '', customer, softwarerelease, maxdepth) -} - -return this; diff --git a/Common_scripts/publish/package.groovy b/Common_scripts/publish/package.groovy deleted file mode 100644 index 68c41f3..0000000 --- a/Common_scripts/publish/package.groovy +++ /dev/null @@ -1,181 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHONBRANCH = "origin/3.6.3_linux" - } - else { - PYTHONBRANCH = "origin/3.6.3_windows" - } - PYTHONDIR = "${WORKSPACE}/python" - PYTHONROOTDIR = 'python' - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) - if (isUnix()) { - env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" - } - else { - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - } - } - } - stage('Clean exports') { - steps { - script { - PACKAGEDEST = "${WORKSPACE}/package" - cleanfolders = [ - PACKAGEDEST, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Copying package artifacts') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - def packageimportoptions = importer.getNewImportOptions() - packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) - packageimportoptions.setincrementversionnumber(true) - packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) - - PACKAGE_FOLDER = "${PACKAGEDEST}/package" - UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" - } - } - } - stage('Aptly environment') { - steps { - script { - echo "Aptly config file is ${APTLY_CONFIG_FILE}" - sh """#!/bin/bash -v - aptly config show - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Create repo - if not already there') { - steps { - script { - echo "Repo name is ${REPO_NAME}" - sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" - if [ \$? -ne 0 ]; then - echo "aptly repo ${REPO_NAME} does not exist; Creating it" - aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - fi - aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Drop published repo - if any') { - steps { - script { - sh """#!/bin/bash -v - # This errors if the published repo exists already; Ignore it - aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" - aptly publish -config="${APTLY_CONFIG_FILE}" list - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - - stage('Add/update package in repo') { - steps { - script { - echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" - sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" - if [ \$? -ne 0 ]; then exit 1; fi - aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Create snapshot') { - steps { - script { - echo "Snapshot name is ${SNAPSHOT_NAME}" - FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" - PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix - echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" - // Convert prefix to regular expression - PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") - PACKAGE_VERSION_REGEX = '' - i = 0 - while(true) { - if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { - PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] - break; - } - PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] - PACKAGE_VERSION_REGEX += '\\.' - i++ - } - sh """#!/bin/bash -v - # This errors if the snapshot exists already; Ignore it - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - - aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' - if [ \$? -ne 0 ]; then exit 1; fi - aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Publish snapshot') { - steps { - script { - sh """#!/bin/bash -v - aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" - if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" list - if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/promptingpublish.groovy b/Common_scripts/publish/promptingpublish.groovy deleted file mode 100644 index ce6a805..0000000 --- a/Common_scripts/publish/promptingpublish.groovy +++ /dev/null @@ -1,182 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean folders') { - steps { - script { - cleanfolders = [ - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Checkout release repo') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - SOURCEURL = RELEASE_REPO - SOURCEREVISION = "origin/master" - SOURCEDIR = "${WORKSPACE}/whole" - configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - } - } - } - stage('Identify customer to publish to') { - steps { - script { - hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") - def List customers = hostedftb.getCustomerList() - timeout(time: 10, unit: 'MINUTES') { - customer = input( - message: 'Please select customer to publish to', - ok: 'Ok', - parameters: [ - choice(name: 'customer', choices: customers, description: 'Which customer?') - ] - ) - echo "Chosen customer is: ${customer}" - } - } - } - } - stage('Identify software release to publish') { - steps { - script { - gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def allreleases = gh.gettaglist(SOURCEDIR) - def releases = new ArrayList() - if (FILTER_REDUNDANT_RELEASES == 'false') { - echo("Showing all releases") - releases = allreleases - } else { - echo("Showing filtered releases") - for (release in allreleases) { - if (!release.toLowerCase().startsWith("superceded") && - !release.toLowerCase().startsWith("superseded") && - !release.toLowerCase().startsWith("erroneous") && - !release.toLowerCase().startsWith("faulty") && - !release.toLowerCase().startsWith("ignore") && - !release.toLowerCase().startsWith("inmainline")) { - echo("Showing release: ${release}") - releases.add(release) - } else { - echo("Not showing release: ${release}") - } - } - } - timeout(time: 10, unit: 'MINUTES') { - release = input( - message: 'Please select the software version', - ok: 'Ok', - parameters: [ - choice(name: 'release', choices: releases, description: 'Which software release?') - ] - ) - echo "Chosen software version is: ${release}" - } - } - } - } - stage('Checkout release repo at selected software version') { - steps { - script { - SOURCEURL = RELEASE_REPO - SOURCEREVISION = "refs/tags/${release}" - SOURCEDIR = "${WORKSPACE}/whole" - configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - } - } - } - stage('Get release artifact info') { - steps { - script { - def mfs = autobuilderscript.getmetafolderscript() - releasemeta = mfs.metafromfolder(SOURCEDIR) - } - } - } - stage('Select which items from software release to publish') { - steps { - script { - publishitems = load("${autobuilder.commonscripts()}/publish/publishitems.groovy") - timeout(time: 10, unit: 'MINUTES') { - def itemsobject = publishitems.createItemSelector( - 'Select software items', - 'Software items', - releasemeta, - SOURCEDIR, - ITEMSDEFAULTLIST, - MAX_META_DEPTH, - '20', - 'PT_MULTI_SELECT') - selecteditems = input( - message: 'Select which items from the release are to be published', - ok: 'Ok', - parameters: [ - itemsobject.multiSelect() - ] - ) - echo "Chosen items are: ${selecteditems}" - } - } - } - } - stage('Publish components') { - steps { - script { - def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, SOURCEDIR, selecteditems, customer, release, MAX_META_DEPTH) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/publish.groovy b/Common_scripts/publish/publish.groovy deleted file mode 100644 index ad33df9..0000000 --- a/Common_scripts/publish/publish.groovy +++ /dev/null @@ -1,92 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) - } - } - } - stage('Clean folders') { - steps { - script { - cleanfolders = [ - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Checkout release repo') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - SOURCEURL = RELEASE_REPO - SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" - SOURCEDIR = "${WORKSPACE}/whole" - SOURCEROOTDIR = 'whole' - configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) - } - } - } - stage('Get release artifact meta') { - steps { - script { - def mfs = autobuilderscript.getmetafolderscript() - releasemeta = mfs.metafromfolder(SOURCEDIR) - } - } - } - stage('Publish components') { - steps { - script { - def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - def hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") - def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/publishitems.groovy b/Common_scripts/publish/publishitems.groovy deleted file mode 100644 index 9f6c368..0000000 --- a/Common_scripts/publish/publishitems.groovy +++ /dev/null @@ -1,129 +0,0 @@ -import com.cwctravel.hudson.plugins.extended_choice_parameter.ExtendedChoiceParameterDefinition - -class PublishableItemsSelection { - def script - def ArrayList preselectedmenuitems - def ArrayList menuitemdescriptions - def ArrayList menuitemsfull - def ExtendedChoiceParameterDefinition multiselect - - PublishableItemsSelection(script) { - this.script = script - preselectedmenuitems = new ArrayList() - menuitemdescriptions = new ArrayList() - menuitemsfull = new ArrayList() - } - - def init( - String name, - String description, - releasemeta, - String releasemetapath, - String preselectedext, - String metalevelsstringext, - String visibleItemCountExt, - String controltype) { - outputSelectionfromPreselect(preselectedmenuitems, preselectedext) - def Integer metalevelsext = metalevelsstringext.toInteger() - def Integer visibleItemCount = visibleItemCountExt.toInteger() - def metaitems = releasemeta.releasewalkerunique(metalevelsext, '', releasemetapath) - for (key in metaitems.keySet()) { - def releaseinfo = metaitems[key] - outputDescription(menuitemdescriptions, releaseinfo) - outputSelection(menuitemsfull, releaseinfo) - } - - script.echo("name is: ${name}") - script.echo("description is: ${description}") - script.echo("menuitemsfull is: ${menuitemsfull}") - script.echo("preselectedmenuitems is: ${preselectedmenuitems}") - script.echo("menuitemdescriptions is: ${menuitemdescriptions}") - script.echo("visibleItemCount is: ${visibleItemCount}") - script.echo("controltype is: ${controltype}") - - def seperator = "," - visibleItemCount = Math.min(visibleItemCount, menuitemsfull.size()) - script.echo("visibleItemCount is now: ${visibleItemCount}") - - multiselect = new ExtendedChoiceParameterDefinition( - name, // String name - controltype, // String type - menuitemsfull.join(seperator), // String value - "project name", // String projectName - "", // String propertyFile - "", // String groovyScript - "", // String groovyScriptFile - "", // String bindings - "", // String groovyClasspath - "", // String propertyKey - preselectedmenuitems.join(seperator),// String defaultValue - "", // String defaultPropertyFile - "", // String defaultGroovyScript - "", // String defaultGroovyScriptFile - "", // String defaultBindings - "", // String defaultGroovyClasspath - "", // String defaultPropertyKey - menuitemdescriptions.join(seperator),// String descriptionPropertyValue - "", // String descriptionPropertyFile - "", // String descriptionGroovyScript - "", // String descriptionGroovyScriptFile - "", // String descriptionBindings - "", // String descriptionGroovyClasspath - "", // String descriptionPropertyKey - "", // String javascriptFile - "", // String javascript - false, // boolean saveJSONParameterToFile - false, // boolean quoteValue - visibleItemCount, // int visibleItemCount - description, // String description - seperator) // String multiSelectDelimiter - } - - def outputSelectionfromPreselect(menuitems, preselectedext) { - script.echo ("preselectext: ${preselectedext}") - def menuitemstemp = preselectedext.split(',') - script.echo ("Menuitemstemp: ${menuitemstemp}") - - for (menuitem in menuitemstemp) { - menuitems.add(menuitem.trim()) - } - } - - def outputDescription(menuitems, releaseinfo) { - def meta = releaseinfo.metainfo - - for (publishableitem in meta.publishableinfo) { - menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") - } - } - - def outputSelection(menuitems, releaseinfo) { - def meta = releaseinfo.metainfo - - for (publishableitem in meta.publishableinfo) { - menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") - } - } - - def multiSelect() { - return multiselect - } -} - -PublishableItemsSelection createItemSelector( - String name, - String description, - releasemeta, - String releasemetapath, - String preselectedext, - String metalevelsstringext, - String visibleItemCountExt = '20', - String type = 'PT_CHECKBOX') { // Could also be "PT_MULTI_SELECT" - PublishableItemsSelection pis = new PublishableItemsSelection(this) - - pis.init(name, description, releasemeta, releasemetapath, preselectedext, metalevelsstringext, visibleItemCountExt, type) - - return pis -} - -return this; diff --git a/Common_scripts/publishing/utils.groovy b/Common_scripts/publishing/utils.groovy deleted file mode 100644 index 2daf6f6..0000000 --- a/Common_scripts/publishing/utils.groovy +++ /dev/null @@ -1,27 +0,0 @@ - - -// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout -def allowed(String filepath) { - if (filepath.endsWith('/**') || filepath.endsWith('/')) { - def files = findFiles glob: filepath - if (files.length > 0) { - return - } else { - error "There are no files/folders under the publishable item: ${filepath}" - } - } - error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" -} - -// Generate a list of artifact files that are publishable to outside parties -def generatelist(meta, publishable) { - filelist = [] - for (filepath in publishable) { - if (filepath != meta.METAARTIFACTS) { - allowed(filepath) - } - meta.addPublishable(filepath) - } -} - -return this; diff --git a/Common_scripts/release/do.groovy b/Common_scripts/release/do.groovy deleted file mode 100644 index 4dcb091..0000000 --- a/Common_scripts/release/do.groovy +++ /dev/null @@ -1,458 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" -repoandtagsfile = "${summaryfolder}/repoandtags.txt" -repoandtagsstring = "" -releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" -tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Release summary: \n") - repoandtagsstring = "" - writeFile(file: repoandtagsfile, text: "Repository summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason += reason + "\n" - echo "Marked build as unstable: Reason: ${reason}" -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def repoandtagsappend(String summary) { - def String repoandtagscontents = readFile(file: repoandtagsfile) - writeFile(file: repoandtagsfile, text: repoandtagscontents + summary) - repoandtagsstring += summary -} - -def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = buildmeta.getNewMetaObject(this) - // See if there is meta data file from the current release - def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" - if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(checkoutfolder) - } - return previousreleasemeta -} - -def checkoutreleaserepo(buildmeta, String checkoutfolder) { - def String releaserepo = buildmeta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - repoandtagsappend(" Release repo: ${releaserepo}") - def previousreleasemeta = buildmeta.getNewMetaObject(this) - dir(releaserepofolder) { - sh("git clone ${releaserepo} ${checkoutfolder}") - } - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) - dir(checkoutfolder) { - // Clear all existing files - Note that this does not remove .git* - sh("git rm -r *") - } - return previousreleasemeta -} - -def checkoutsourcerepo(buildmeta, String checkoutfolder) { - def String sourcerepo = buildmeta.gitinfo.remoteurl - if (sourcerepo) { - repoandtagsappend(" Source repo: ${sourcerepo}") - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } - } else { - repoandtagsappend(" Source repo: None") - } -} - -def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { - // Copy relevant artifacts to temp folder - summaryappend("Artifacts: ") - def boolean result = true - for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { - def filterpath = artifactfilter - if (buildinfo.accumulatedpath) { - filterpath = buildinfo.accumulatedpath + '/' + artifactfilter - } - // Copy relevant artifacts to temp folder - echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - try { - copyArtifacts( - filter: filterpath, - projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - ], - target: tempfolder - ) - } catch (Exception e) { - echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" - result = false - summaryappend("Does not exist:") - } - if (result) { - // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath - dir(originalrootdir) { - sh("mv -v * ${repofolder}") - } - } - summaryappend("${artifactfilter},") - } - return result -} - -def releasegitadd() { - // Add anything new - sh("git add .") - summaryappend("Add,") -} - -def releasegitreset() { - // Add anything new - sh("git reset --hard") - sh("git clean -fdx") - summaryappend("Reset,") -} - -def releasegitcommit(String buildstamp) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") -} - -// Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { - String buildsha = buildmeta.gitinfo.gitrevision - String previoussha = previousreleasemeta.gitinfo.gitrevision - - def boolean shachanged = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" - - String buildstamp = buildmeta.buildstampinfo.buildstamp - String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp - - def boolean buildstampchanged = (buildstamp != previousbuildstamp) - echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" - - def boolean changed = (shachanged | buildstampchanged) - echo "changed: ${changed}" - - return changed -} - -def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { - def ArrayList taglist = new ArrayList() - taglist.add(buildstamp) - - def ArrayList othertaglist = otherreleasetags.split(',') - for (othertagitem in othertaglist) { - if (othertagitem?.trim()) { - def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" - taglist.add(componentothertagitem) - } - } - return taglist -} - -def String releasegitgetshacode(String tag) { - def existingshacode = "" - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - existingshacode = readFile(file: gitrevparseoutputfile).trim() - } - return existingshacode -} - -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def String existingshacode = releasegitgetshacode(tag) - if (existingshacode) { // Tag exists - if (existingshacode != postcommitshacode) { // If the existing tag is not associated with the one expected - // This is a problem - The tag is already at another position in git repo - setunstable("The release tag ${tag} has been used at another commit other than one expected - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${postcommitshacode})") - summaryappend("(${tag} - Could not add!)") - } else { - summaryappend("(${tag} - Already added)") - } - } else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\"") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def boolean releasegittags(String postcommitshacode, ArrayList taglist) { - boolean tagadded = false - - summaryappend("Rel Tag ") - for (tag in taglist) { - if (releasegittag(postcommitshacode, tag)) { - echo "Tag added: ${tag} with sha: ${postcommitshacode}" - tagadded = true - } - } - return tagadded -} - -def boolean sourcegittag(String shacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != shacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${shacode})") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\" ${shacode}") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def sourcegittags(String buildsha, List taglist) { - boolean tagadded = false - - summaryappend(",Src Tag ") - for (tag in taglist) { - if (sourcegittag(buildsha, tag)) { - tagadded = true - } - } - return tagadded -} - -def releasegitpush(String dryrunpush ) { - sh("git remote -v") - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend(",Dry-Run-Push") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend(",Push") - } -} - -def sourcegitpush(String dryrunpush ) { - sh("git remote -v") - if (dryrunpush == "true") { - sh("git push -n --tags") - summaryappend(",Src-Dry-Run-Push") - setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags") - summaryappend(",Src Push") - } -} - -def boolean isalreadyreleased(buildmeta, tag) { - def boolean alreadyreleased = buildmeta.releaseinfo.released - if (alreadyreleased) { - echo "Component is already marked as released: ${alreadyreleased}" - } else { - String commitshacode = releasegitgetshacode(tag) - if (commitshacode) { - echo "Tag: ${tag} already exists in this repo therefore marking as already released" - alreadyreleased = true - } else { - alreadyreleased = false - } - } - return alreadyreleased -} - -def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { - summaryappend(" Git operations: ") - def String buildstamptag = buildmeta.releasetag() - - dir(releaserepo) { - def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Pre commit sha code: ${precommitshacode}" - - def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) - - def String postcommitshacode - def boolean changed = false - if (alreadyreleased) { - summaryappend("Already-released,") - postcommitshacode = releasegitgetshacode(buildstamptag) - echo "Previous release sha code: ${postcommitshacode}" - } else { - releasegitadd() - - changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamptag) - } else { - releasegitreset() - summaryappend("No-commit,") - } - - postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" - } - - ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) - def tagadded = releasegittags(postcommitshacode, taglist) - - if (changed || tagadded) { - releasegitpush(dryrunpush) - } - } - dir(sourcerepo) { - def String srcbuildstamptag = buildmeta.globalreleasetag() - def String buildsha = buildmeta.gitinfo.gitrevision - - if (buildsha) { - def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) - def tagadded = sourcegittags(buildsha, srctaglist) - if (tagadded) { - sourcegitpush(dryrunpush) - } - } else { - summaryappend(",No Src ") - } - } -} - -def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { - def buildmeta = buildinfo.metainfo - def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) - def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - - summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") - repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") - def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) - - checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - - if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) - } - - summaryappend("\n") - repoandtagsappend("\n") -} - -def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { - def int md = maxdepth.toInteger() - resetfolders() - preconditions() - - // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) - echo "Unique components:" - def String versioncomponent = "" - for (key in componentReleaseInfo.keySet()) { - versioncomponent += "${key}\n" - } - echo versioncomponent - - // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) - def String versionuniquecomponent = '' - def String versionwithpathscomponent = '' - for (key in componentReleaseInfoVersionUnique.keySet()) { - versionuniquecomponent += "${key}\n" - versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" - } - echo 'Version unique components:' - echo versionuniquecomponent - - echo 'Version unique components with paths:' - echo versionwithpathscomponent - - if (DO_INTEGRITY_CHECK == "true") { - if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { - error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" - } - } - - for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { - error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" - } - } - - def toplevelid = buildmeta.componentidentifier() - // For each component - for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - echo "Releasing component: ${buildinfo.metainfo.componentidentifier()}" - if (buildinfo.metainfo.componentidentifier() != '___' && buildinfo.metainfo.componentidentifier() != '_') { - if (toplevelid != key) { - releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) - } - } - } - // Do the top level last - def toplevelbuildinfo = componentReleaseInfo[toplevelid] - releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) - - return [summarystring, markunstable, unstablereason] -} - -return this; diff --git a/Common_scripts/release/registerrepo.groovy b/Common_scripts/release/registerrepo.groovy deleted file mode 100644 index f565706..0000000 --- a/Common_scripts/release/registerrepo.groovy +++ /dev/null @@ -1,12 +0,0 @@ - -vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo) { - if (releaserepo?.trim()) { - vc.checkRepoExists(releaserepo) - } - meta.releaseinfo.set(releaserepo) -} - -return this; diff --git a/Common_scripts/release/release.groovy b/Common_scripts/release/release.groovy deleted file mode 100644 index 82af511..0000000 --- a/Common_scripts/release/release.groovy +++ /dev/null @@ -1,87 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - cleanfolders = [ - "${WORKSPACE}/build_artifacts", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Get build artifact meta') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - BUILD_ARTIFACTS = 'build' - - def importoptions = importer.getNewImportOptions() - importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) - importoptions.setincrementversionnumber(false) - importoptions.setfilter('meta/**') - importoptions.setappendtoparentmeta(false) - buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) - } - } - } - stage('Release components') { - steps { - script { - def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") - // Place the top level build stamp on all - OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" - (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/remoteaccess/ftp.groovy b/Common_scripts/remoteaccess/ftp.groovy deleted file mode 100644 index 25b9143..0000000 --- a/Common_scripts/remoteaccess/ftp.groovy +++ /dev/null @@ -1,32 +0,0 @@ -import org.apache.commons.net.ftp.FTPClient - -class FTPSiteInfo { - String url - String username - String password - String workingdirectory - - FTPSiteInfo(String url, String username, String password, String workingdirectory) { - this.url = url - this.username = username - this.password = password - this.workingdirectory = workingdirectory - } -} - -FTPSiteInfo createFTPSiteInfo(String url, String username, String password, String workingdirectory) { - FTPSiteInfo ftpsiteinfo = new FTPSiteInfo(url, username, password, workingdirectory) - return ftpsiteinfo -} - -FTPClient createFTPClient(FTPSiteInfo ftpsiteinfo) { - FTPClient ftpclient = new FTPClient() - ftpclient.connect(ftpsiteinfo.url) - ftpclient.enterLocalPassiveMode() - ftpclient.login(ftpsiteinfo.username, ftpsiteinfo.password) - ftpclient.changeWorkingDirectory(ftpsiteinfo.workingdirectory) - - return ftpclient -} - -return this; diff --git a/Common_scripts/remoteaccess/hostedftp.groovy b/Common_scripts/remoteaccess/hostedftp.groovy deleted file mode 100644 index cf62d32..0000000 --- a/Common_scripts/remoteaccess/hostedftp.groovy +++ /dev/null @@ -1,28 +0,0 @@ -ftpremote = load("${autobuilder.commonscripts()}/remoteaccess/ftp.groovy") - -def getHostedFTPSiteInfo() { - hostedftpsiteinfo = - ftpremote.createFTPSiteInfo("ftp.hostedftp.com", - "ftp@globalinkjetsystems.com", - "ramshorn42", - "/Customers/") - return hostedftpsiteinfo -} - -def getCustomerList() { - def List customerfilenames = new ArrayList() - def List customers = new ArrayList() - - def ftpsiteinfo = getHostedFTPSiteInfo() - def ftpclient = ftpremote.createFTPClient(ftpsiteinfo) - - customerfilenames = ftpclient.listNames() - for(customer in customerfilenames) { - def String converted = new String(customer.getBytes("ISO-8859-1"), "UTF-8") - customers.add(converted) - } - ftpclient.disconnect() - return customers -} - -return this; \ No newline at end of file diff --git a/Common_scripts/staticanalysis/ccm.groovy b/Common_scripts/staticanalysis/ccm.groovy deleted file mode 100644 index ae91cb0..0000000 --- a/Common_scripts/staticanalysis/ccm.groovy +++ /dev/null @@ -1,85 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - CCMURL = 'git@hydra:tools/ccm.git' - CCMBRANCH = 'origin/master' - CCMDIR = "${WORKSPACE}/ccm" - configurationhelper.gitCheckoutAndRegister(meta, 'ccm', CCMURL, CCMBRANCH, CCMDIR) - CCM = "${CCMDIR}/bin/CCM.exe" - } - } - } - stage('Checkout source') { - steps { - script { - SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) - } - } - } - stage("Clean exports") { - steps { - script { - RESULTS = 'results' - cleanfolders = [ - RESULTS, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Static analysis') { - steps { - script { - def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" - writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) - echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" - - dir(RESULTS) { - echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" - if (isUnix()) { - sh """#!/bin/bash - ${CCM}" "${ccmconfigurationfile}" > ccm.xml - """ - } else { - echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" - bat """ - ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml - """ - } - } - recordIssues(tools: [ - ccm(pattern: "**/ccm.xml", reportEncoding: 'UTF-8'), - ]) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/staticanalysis/cpd.groovy b/Common_scripts/staticanalysis/cpd.groovy deleted file mode 100644 index c04649c..0000000 --- a/Common_scripts/staticanalysis/cpd.groovy +++ /dev/null @@ -1,152 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHONBRANCH = 'origin/3.6.3_linux' - } else { - PYTHONBRANCH = 'origin/3.6.3_windows' - } - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - - JAVAURL = 'git@hydra:tools/openjdk.git' - if (isUnix()) { - JAVABRANCH = 'origin/openjdk-11.0.2_linux-x64_bin' - } else { - JAVABRANCH = 'origin/openjdk-11.0.2_windows-x64_bin' - } - JAVADIR = "${WORKSPACE}/java" - configurationhelper.gitCheckoutAndRegister(meta, 'java', JAVAURL, JAVABRANCH, JAVADIR) - JAVA = "${JAVADIR}/ver/bin" - if (isUnix()) { - env.PATH = JAVA + ':' + PATH - } else { - env.PATH = JAVA + ';' + PATH - } - - PPDURL = 'git@hydra:tools/pmd.git' - PPDBRANCH = 'origin/master' - PPDDIR = "${WORKSPACE}/ppd" - configurationhelper.gitCheckoutAndRegister(meta, 'ppd', PPDURL, PPDBRANCH, PPDDIR) - PPD = "${PPDDIR}/bin" - } - } - } - stage('Checkout source') { - steps { - script { - SOURCEREVISION = "${SOURCE_REVISION}" - SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - SRC = "${SOURCE_PATH}" - if (SOURCE_OFFSET_PATH) { - SRC += "/${SOURCE_OFFSET_PATH}" - } - } - } - } - stage("Clean exports") { - steps { - script { - STATICANALYSIS = 'staticanalysis' - cleanfolders = [ - STATICANALYSIS, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } - stage('Static analysis') { - steps { - script { - def PPDFOLDER = "${STATICANALYSIS}/ppd" - def EXCLUDE_PARAM = '' - if (EXCLUDE_PATHS) { - EXCLUDE_PARAM = '--exclude "' - def exclude_path_list = EXCLUDE_PATHS.split(',') - for (exclude_path in exclude_path_list) { - EXCLUDE_PARAM += "${SRC}/${exclude_path.trim()}," - } - EXCLUDE_PARAM += '"' - } - if (isUnix()) { - sh """#!/bin/bash - set -ev - - mkdir -p "${PPDFOLDER}" - export PMD_JAVA_OPTS=-Xmx65536m - "${PPD}/run.sh" cpd --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" - """ - } else { - bat """ - mkdir -p "${PPDFOLDER}" - set PMD_JAVA_OPTS=-Xmx2048m - "${PPD}/cpd.bat" --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" - """ - } - recordIssues(tools: [ - cpd(pattern: "**/cpd.xml", reportEncoding: 'UTF-8'), - ]) - } - } - } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/staticanalysis/cppcheck.groovy b/Common_scripts/staticanalysis/cppcheck.groovy deleted file mode 100644 index d959770..0000000 --- a/Common_scripts/staticanalysis/cppcheck.groovy +++ /dev/null @@ -1,114 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - PYTHONBRANCH = 'origin/3.6.3_windows' - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - } - } - stage('Checkout source') { - steps { - script { - SOURCEREVISION = "${SOURCE_REVISION}" - SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" - } - } - } - stage("Clean exports") { - steps { - script { - STATICANALYSIS = 'staticanalysis' - cleanfolders = [ - STATICANALYSIS, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } - stage('Static analysis') { - steps { - script { - def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" - dir(CPPCHECKFOLDER) { - echo "Executing: ${CPPCHECK} --xml --xml-version=2 ${SRC} 2> cppcheck.xml" - if (isUnix()) { - sh """#!/bin/bash - set -e - - "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml - """ - } else { - bat """ - "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml - """ - } - } - recordIssues(tools: [ - cppCheck(pattern: "**/cppcheck.xml", reportEncoding: 'UTF-8'), - ]) - } - } - } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/test/testethernetproduct.groovy b/Common_scripts/test/testethernetproduct.groovy deleted file mode 100644 index 5e2ce53..0000000 --- a/Common_scripts/test/testethernetproduct.groovy +++ /dev/null @@ -1,204 +0,0 @@ -import jenkins.model.Jenkins - -pipeline { - agent { label AGENT_LABEL } - stages { - stage ("Initial Setup") { - steps{ - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage ("Checkout test environment") { - steps { - script { - toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") - - def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' - def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' - def TEST_ENVIRONMENT_REVISION = 'origin/master' - - def environmentoptions = toolsimporter.getNewToolOptions() - environmentoptions.toolName('Ethernet product test environment') - environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) - toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) - - TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" - def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") - TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] - echo "Selected environment:\n${TEST_ENVIRONMENT}" - } - } - } - stage ('Setup') { - agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } - stages { - stage ("Checkout dependencies") { - steps { - script { - def PYTHON_URL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHON_REVISION = 'origin/3.6.3_linux' - } - else { - PYTHON_REVISION = 'origin/3.8.10_windows_for_test' - } - PYTHONDIR = "${WORKSPACE}/python" - - def pythonimportoptions = toolsimporter.getNewToolOptions() - pythonimportoptions.toolName('Python') - pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) - toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) - if (isUnix()) { - env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" - } - else { - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - - def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' - TEST_SUITE_DIR = 'TestSuiteDir' - env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" - - def regressiontestimportoptions = toolsimporter.getNewToolOptions() - regressiontestimportoptions.toolName('Tests') - regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) - toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) - - env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" - - def resultimportoptions = toolsimporter.getNewToolOptions() - resultimportoptions.toolName('Test history') - resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) - toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) - - def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' - env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" - - def coreimportoptions = toolsimporter.getNewToolOptions() - coreimportoptions.toolName('Test core') - coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) - toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) - - def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' - def FTDI_SOURCE_REVISION = 'origin/Core-98' - FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" - - def ftdiimportoptions = toolsimporter.getNewToolOptions() - ftdiimportoptions.toolName('FTDI relay driver') - ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) - toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) - - def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' - HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" - - def resultprocessingoptions = toolsimporter.getNewToolOptions() - resultprocessingoptions.toolName('History processing') - resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) - toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) - } - } - } - stage ("Import application to test") { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - APPLICATION_UNDER_TEST = 'application_under_test' - env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" - - def applicationimportoptions = importer.getNewImportOptions() - applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) - applicationimportoptions.setfilter('app_package/**,control_package/**') - applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) - - meta.productinfo.set(applicationmeta.productinfo) - meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" - meta.gitinfo.set(applicationmeta.gitinfo) - meta.buildstampinfo.set(applicationmeta.buildstampinfo) - } - } - } - stage("Clean") { - steps { - script { - LOGGING_ROOT = 'Logging' - env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" - cleanfolders = [ - GIS_TEST_OUTPUT_LOGGING_ROOT, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage ('Test') { - steps { - script { - lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { - if (POWER_CYCLE_HARDWARE == 'true') { - dir(FTDI_SOURCE_DIR) { - bat """ - "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True - """ - } - } - if (UPDATE_HARDWARE == 'true') { - dir(GIS_TEST_SUITE_CORE) { - bat """ - "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" - """ - rc = bat (script:""" - "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" - """, - returnStatus: true) - if (rc == 1) { - unstable('FAILED while inspecting state of hardware') - } - } - } - if (DO_TEST == 'true') { - rc = bat(script:""" - ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} - """, - returnStatus: true - ) - if (rc == 1) { - unstable('FAILED while running tests') - } - } - } - } - } - } - stage("Archive Test History") { - steps{ - script { - dir(HISTORY_FOLDER) { - rc = bat (script:""" - "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" - """, - returnStatus: true - ) - if (rc == 1) { - unstable('FAILED to process history') - } - } - } - } - } - stage("Archive test artifacts"){ - steps{ - script { - def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - } - } -} - diff --git a/Common_scripts/textutils/subst.groovy b/Common_scripts/textutils/subst.groovy deleted file mode 100644 index c0207a3..0000000 --- a/Common_scripts/textutils/subst.groovy +++ /dev/null @@ -1,8 +0,0 @@ - -def substitute(String filepath, pattern, String subst) { - fileText = readFile(file: filepath, encoding: 'UTF-8') - def converted = fileText.replaceAll(pattern, subst) - writeFile(file: filepath, text: converted, encoding: 'UTF-8') -} - -return this; \ No newline at end of file diff --git a/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig b/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig deleted file mode 100644 index ea618a3..0000000 --- a/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig +++ /dev/null @@ -1,25 +0,0 @@ -{ - "version": "1.0", - "components": [ - "Microsoft.VisualStudio.Workload.ManagedDesktop", - "Microsoft.VisualStudio.Workload.NativeDesktop", - "Microsoft.VisualStudio.Workload.NetWeb", - "Microsoft.VisualStudio.Workload.NetCoreTools", - "microsoft.net.componentgroup.targetingpacks.common", - "microsoft.componentgroup.blend", - "microsoft.visualstudio.component.entityframework", - "microsoft.visualstudio.component.debugger.justintime", - "microsoft.visualstudio.component.wcf.tooling", - "microsoft.visualstudio.component.vc.diagnostictools", - "microsoft.visualstudio.component.vc.cmake.project", - "microsoft.visualstudio.component.vc.testadapterforboosttest", - "microsoft.visualstudio.component.vc.testadapterforgoogletest", - "microsoft.visualstudio.component.vc.atlmfc", - "microsoft.visualstudio.component.appinsights.tools", - "microsoft.visualstudio.component.diagnostictools", - "microsoft.visualstudio.componentgroup.web.cloudtools", - "microsoft.visualstudio.component.aspnet45", - "microsoft.visualstudio.component.webdeploy", - "microsoft.visualstudio.component.nuget.buildtools" - ] -} \ No newline at end of file diff --git a/Common_scripts/versioncontrol/git.groovy b/Common_scripts/versioncontrol/git.groovy deleted file mode 100644 index a22054b..0000000 --- a/Common_scripts/versioncontrol/git.groovy +++ /dev/null @@ -1,377 +0,0 @@ -import hudson.plugins.git.* -import hudson.plugins.git.extensions.* -import hudson.plugins.git.extensions.impl.* - -clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") - -def boolean isShallowClonedRepo(String gitrootpath) { - def boolean isshallowclone = true - - dir( gitrootpath) { - if (isUnix()) { - isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() - } - else { - isshallowclone = powershell(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() - } - echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" - } - return isshallowclone -} - -String getGitInfo(String workingSetRootPath) { - dir(workingSetRootPath) { - return clh.getCommandOutput('git rev-parse HEAD') - } -} - -ArrayList gettaglist(String workingSetRootPath) { - ArrayList taglist = new ArrayList() - dir(workingSetRootPath) { - def tagliststring = clh.getCommandOutput('git tag --sort=-version:refname') - taglist = tagliststring.split('\\n') - } - return taglist -} - -@NonCPS -def ArrayList sparsecheckoutconfig(ArrayList paths) { - ArrayList sparsecheckoutpaths = new ArrayList() - for (path in paths) { - sparsecheckoutpaths.add(new SparseCheckoutPath(path)) - } - return sparsecheckoutpaths -} - -def String uniquereporootname(String gitURL) { - def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") - echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" - return uniquename -} - -// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo -def String checkouttoreporoot(String gitURL) { - def String reporootsubfolder - def String reporootenv = env.REPO_ROOT - if (!reporootenv) { - reporootsubfolder = "" - echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" - } else { - def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootname}" - - if (!fileExists(reporootsubfolder)) { - echo "Checking out ${gitURL} to ${reporootsubfolder}" - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: 'origin/master'] - ], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CheckoutOption', timeout: 180], - [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], - ], - submoduleCfg: [], - ]) - } else { - echo "Found ${gitURL} checked out at ${reporootsubfolder}" -/* - dir(reporootsubfolder) { - if (isUnix()) { - sh """ - git status - if [ \$? -ne 0 ]; then exit 1; fi - """ - } else { - powershell """ - git status - if (\$lastexitcode -ge 1) { exit 1 } - """ - } - } -*/ - } - } - return reporootsubfolder -} - -def checkgitlfsinstall() { - if (isUnix()) { - sh """ - git lfs install - if [ \$? -ne 0 ]; then exit 1; fi - """ - - } else { - powershell """ - git lfs install - if (\$lastexitcode -ge 1) { exit 1 } - """ - } -} - -//No credentials specified -//Cloning the remote Git repository -//Cloning repository git@hydra:print-server-releases/print_server_whole.git -//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 -//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git -//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git -// > git --version # timeout=10 -// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 -// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 -// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 -// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 -// Cleaning workspace -// > git rev-parse --verify HEAD # timeout=10 -// No valid HEAD. Skipping the resetting -// > git clean -fdx # timeout=10 -// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git -// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 -// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 -// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 -// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) -// Enabling Git LFS pull -// > git config core.sparsecheckout # timeout=10 -// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 -// > git config --get remote.origin.url # timeout=10 -// > git lfs pull origin # timeout=120 -// Commit message: "2.10180.7389.21125_7" -// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 - - -// > git rev-parse --is-inside-work-tree # timeout=10 -// Fetching changes from the remote Git repository -// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 -// Cleaning workspace -// > git rev-parse --verify HEAD # timeout=10 -// Resetting working tree -// > git reset --hard # timeout=10 -// > git clean -fdx # timeout=10 -// Using shallow fetch with depth 1 -// Fetching upstream changes from git@hydra:releases/print-server-installer.git -// > git --version # timeout=10 -// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 -// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 -// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 -// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) -// Enabling Git LFS pull -// > git config core.sparsecheckout # timeout=10 -// > git read-tree -mu HEAD # timeout=10 -// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 -// > git config --get remote.origin.url # timeout=10 -// > git lfs pull origin # timeout=120 -// Commit message: "2.10180.7389.21125_3" -// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 - -def String lfsinstallandreporoot(String gitURL) { - checkgitlfsinstall() - return checkouttoreporoot(gitURL) -} - -def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { - if (!sparseitemlist.isEmpty()) { - def itemstring = "Items in sparse item list:\n" - for (item in sparseitemlist) { - itemstring += item + "\n" - } - echo itemstring - } -} - -def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { - echo "Git: Checking out: URL: ${gitURL} Revision: ${revision} to: ${targetDir}" - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: revision] - ], - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], - [$class: 'PruneStaleBranch'], - ], - submoduleCfg: [], - ]) -} - -// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - def String reporootpath = lfsinstallandreporoot(gitURL) - showsparseitemlist(sparseitemlist) - docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) -} - -def gitinit(String targetDir, String reporootpath) { - if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist - echo "git init ${targetDir}" - if (isUnix()) { - sh """ - git init "${targetDir}" - """ - } else { - powershell """ - git init "${targetDir}" - """ - } - } - echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" - dir("${targetDir}/.git/objects/info") { - writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") - } -} - -def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { - def String sparseitemliststring = "" - if (!sparseitemlist.isEmpty()) { - for (item in sparseitemlist) { - sparseitemliststring += "${item}," - } - echo "git config lfs.fetchinclude ${sparseitemliststring}" - dir(targetDir) { - if (isUnix()) { - sh """ - git config lfs.fetchinclude "${sparseitemliststring}" - """ - } else { - powershell """ - git config lfs.fetchinclude "${sparseitemliststring}" - """ - } - } - } -} - -def gitsparserestore(String targetDir) { - echo "Clean out full fetch; Force a sparse fetch" - dir(targetDir) { - if (isUnix()) { - sh """ - git rm -rf . > /dev/null - git clean -fxd > /dev/null - git reset --hard > /dev/null - """ - } else { - powershell """ - git rm -rf . > \$null - git clean -fxd > \$null - git reset --hard > \$null - """ - } - } - } - -def List convertSparseItemsString(String itemstopublish) { - def List sparsepaths = [] - - boolean nonempty = itemstopublish?.trim() - if (nonempty) { - def String[] items = itemstopublish.trim().split(',') - def String allpaths = "" - for(String item : items) { - sparsepaths.add(item) - allpaths += "${item}\n" - } - echo "Selected item to publish:\n${allpaths}" - } - return sparsepaths -} - -def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - echo "Performing alternative checkout" - def String reporootpath = lfsinstallandreporoot(gitURL) - showsparseitemlist(sparseitemlist) - if (!sparseitemlist.isEmpty()) { - gitinit(targetDir, reporootpath) - setlfsfetchinclude(targetDir, sparseitemlist) - } - docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) - if (!sparseitemlist.isEmpty()) { - gitsparserestore(targetDir) - } -} - -def checkRepoExists(String gitURL) { - def int result = 0 - if (isUnix()) { - result = sh( script: """ - git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - """, - returnStatus: true - ) - } else { - result = powershell( script: """ - git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - """, - returnStatus: true - ) - } - echo "git ls-remote return result: ${result}" - if (result > 0) { - error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") - } else { - echo "The release repo: ${gitURL} exists" - } -} - -String getSHAFromVersion(String versionFolder) { - dir (versionFolder) { - srcRevision = readFile("gitrevision.txt").trim() - } - return srcRevision -} - -boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { - def keys = versions.keySet() - if (keys.size() < 1) { - return false - } - if (keys.size() == 1) { - return true - } - else { - version = "" - for (key in keys) { - nextversion = versions.get(key) - if (version == "") { - version = nextversion - } - if (nextversion != version) { - return false - } - } - } - return true -} - -def stripgithistory(String rootfolder) { - echo "Stripping git history out of ${rootfolder}" - dir(rootfolder) { - if (isUnix()) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rf XX - find . -name ".gitignore" | xargs -iXX rm -rf XX - find . -name ".gitmodules" | xargs -iXX rm -rf XX - find . -iname ".gitattributes" | xargs -iXX rm -rf XX - find . -iname "*@tmp" | xargs -iXX rm -rf XX - """ - } else { - echo "Not supporting git stripping in Windows at the moment" - } - } -} - -return this diff --git a/Common_scripts/versionnumbering/do.groovy b/Common_scripts/versionnumbering/do.groovy deleted file mode 100644 index 208a60f..0000000 --- a/Common_scripts/versionnumbering/do.groovy +++ /dev/null @@ -1,23 +0,0 @@ - -def stamp(convertdetails) { - for (convertdetail in convertdetails) { - def filetoconvert = convertdetail[0] - def pattern = convertdetail[1] - def replacement = convertdetail[2] - echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" - def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - echo "Using python: ${PYTHON} and script: ${stampscript}" - if (isUnix()) { - sh """ - "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" - """ - } - else { - powershell """ - & "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" - """ - } - } -} - -return this; diff --git a/Common_scripts/versionnumbering/generate.groovy b/Common_scripts/versionnumbering/generate.groovy deleted file mode 100644 index 5e974e2..0000000 --- a/Common_scripts/versionnumbering/generate.groovy +++ /dev/null @@ -1,180 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -def int maxversionnumber() { - def int MAXVERSIONNUMBER = 65535 - return MAXVERSIONNUMBER -} - -int incrementrevision(int current, int increment) { - def int newrevision = (current + increment) % (maxversionnumber() + 1) - echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" - return newrevision -} - -def getGitRevision(String gitrootpath, int basebuildnumber) { - def int adjustedchangecount - def String gitrevision - def String remoteinfo - def String remoteurl - - def boolean isShallowCloned = gh.isShallowClonedRepo(gitrootpath) - if (isShallowCloned) { - error("Cannot generate a version number from a shallow cloned repo at ${gitrootpath}") - } - - dir( gitrootpath) { - def int changecount = 0 - if (isUnix()) { - changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() - } - else { - changecount = powershell(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() - } - echo "Change count: ${changecount}" - def int mvnp1 = maxversionnumber() + 1 - basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber - adjustedchangecount = incrementrevision(changecount, basebuildnumber) - echo "Adjusted change count: ${adjustedchangecount}" - if (isUnix()) { - gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() - } - else { - gitrevision = powershell(returnStdout: true, script: "git rev-parse HEAD").trim() - } - echo "Revision: ${gitrevision}" - if (isUnix()) { - remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() - } - else { - remoteinfo = powershell(returnStdout: true, script: "git remote show origin").trim() - } - echo "Remote info: ${remoteinfo}" - if (isUnix()) { - remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() - } - else { - remoteurl = powershell(returnStdout: true, script: "git config --get remote.origin.url").trim() - } - echo "Remote URL: ${remoteurl}" - } - return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] -} - -// Handle: -// origin/master -// origin/2.0.x -def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { - echo "Determining version number prefix from source branch name: ${sourcebranch}" - sourcebranch = sourcebranch.toLowerCase() - - def defaultversionprefix = "0.30000.0." - def splits = sourcebranch.split('/') - if (splits.size() < 1) { - echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" - return defaultversionprefix - } - - def mastertypeversionprefix = "0.20000.0." - sourcebranch = splits.last() - if (sourcebranch.toLowerCase().find("master") != null) { - echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" - return mastertypeversionprefix - } - - def splits2 = sourcebranch.split('\\.') - if (splits2.size() < 1) { - def thirdpartyversionprefix = "0.25000.0." - echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" - return thirdpartyversionprefix - } - - for (split in splits2) { - if (split == "x") { - echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" - return mastertypeversionprefix - } - } - - def integerstrings = sourcebranch.findAll(/\d+/) - def integers = [] - for (integerstring in integerstrings) { - if (integerstring.length() > 6) { - integerstring = integerstring.substring(0, 5) - } - int integer = integerstring.toInteger() - integer = integer % (maxversionnumber() + 1) - integers.add(integer) - } - - def versionprefix = "" - if (integers.size() < 1) { - echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" - versionprefix = defaultversionprefix - } - else { - def int integernumbercount = 0 - for (integer in integers) { - versionprefix += integer + "." - integernumbercount += 1 - if (integernumbercount > 2) { - break - } - } - if (integernumbercount == 0) { - echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" - versionprefix = defaultversionprefix - } - else if (integernumbercount == 1) { - versionprefix += "0.0." - } - else if (integernumbercount == 2) { - versionprefix += "0." - } - } - if (droptrailingdot == true) { - versionprefix = versionprefix.substring(0, versionprefix.length() - 1); - } - echo "Determined version prefix to be: ${versionprefix}" - return versionprefix -} - -String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { - def int gitrevisioncount - def String gitrevision - def String remoteinfo - def String remoteurl - - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) - - def versionprefix = getVersionPrefix(sourcerevision) - - def buildstamp = versionprefix + gitrevisioncount.toString() - echo "Build stamp is: ${buildstamp}" - - meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) - - return buildstamp -} - -String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { - generate(meta, sourcerevision, sourcepath, basebuildnumber) - meta.buildstampinfo.override(pregeneratedbuildstamp) - - return pregeneratedbuildstamp -} - -String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { - def int bbn = basebuildnumber.trim().toInteger() - return generate(meta, sourcerevision, sourcepath, bbn) -} - -String copyfrom(meta, othermeta) { - // Deep copy - meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) - meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) - - return meta.buildstampinfo.buildstamp -} - -return this; \ No newline at end of file diff --git a/Common_scripts/versionstamping/.project b/Common_scripts/versionstamping/.project deleted file mode 100644 index 74928d6..0000000 --- a/Common_scripts/versionstamping/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - versionstamping - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/Common_scripts/versionstamping/.pydevproject b/Common_scripts/versionstamping/.pydevproject deleted file mode 100644 index 746d0f0..0000000 --- a/Common_scripts/versionstamping/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - - -/${PROJECT_DIR_NAME} - -python 3.0 -python35 - diff --git a/Common_scripts/versionstamping/versionstamper.py b/Common_scripts/versionstamping/versionstamper.py deleted file mode 100644 index 9243aa4..0000000 --- a/Common_scripts/versionstamping/versionstamper.py +++ /dev/null @@ -1,38 +0,0 @@ -import argparse -import sys -import shutil -from builtins import FileNotFoundError - -def main(argv): - parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") - - # Call once arguments to allow or disallow something - parser.add_argument('-f', '--filetoconvert', help='The file to convert') - parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace') - parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') - - args = parser.parse_args(argv) - - filetoconvert = args.filetoconvert - pattern = args.pattern - substitute = args.substitute - - print("Converting instances of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) - - with open(filetoconvert,'r') as c: - fileread = c.read() - - count = 0 - with open(filetoconvert, 'w') as f: - print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) - converted = fileread.replace(pattern, substitute) - count = converted.count(substitute) - print("Confirmed %s instances of %s post-conversion"%(count,substitute)) - f.write(converted) - - if count == 0: - return 1 - return 0 - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:])) From 11826a463cc876fd54dd8b7c52274b3e6e64c4ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 19:19:43 +0000 Subject: [PATCH 08/72] Get list of tags for each product release repo --- versioncontrol/git.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a22054b..8a26fb7 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,6 +34,13 @@ ArrayList gettaglist(String workingSetRootPath) { return taglist } +ArrayList getremotetaglist(String repositoryURL) { + ArrayList taglist = new ArrayList() + def tagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") + taglist = tagliststring.split('\\n') + return taglist +} + @NonCPS def ArrayList sparsecheckoutconfig(ArrayList paths) { ArrayList sparsecheckoutpaths = new ArrayList() From c5f01d0f1a76396d056f1af8d2e6346130f18044 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 20:22:51 +0000 Subject: [PATCH 09/72] Build up a tag map --- versioncontrol/git.groovy | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8a26fb7..7cfd20f 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,11 +34,19 @@ ArrayList gettaglist(String workingSetRootPath) { return taglist } -ArrayList getremotetaglist(String repositoryURL) { +LinkedHashMap> getremoteshatagmap(String repositoryURL) { ArrayList taglist = new ArrayList() - def tagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") - taglist = tagliststring.split('\\n') - return taglist + def shatagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") + def shataglist = tagliststring.split('\\n') + + LinkedHashMap> shatagmap = new LinkedHashMap>() + for (shatag in shataglist) { + def shatagfields = shatag.split(' ') + def sha = shatagfields[0].trim() + def tag = shatagfields[1].trim() + shatagmap[sha].add(tag) + } + return shatagmap } @NonCPS From 453a14acc597f40bbbe117c58ccba05af8f72158 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 20:25:17 +0000 Subject: [PATCH 10/72] Build up a tag map --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 7cfd20f..25221e0 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -37,7 +37,7 @@ ArrayList gettaglist(String workingSetRootPath) { LinkedHashMap> getremoteshatagmap(String repositoryURL) { ArrayList taglist = new ArrayList() def shatagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") - def shataglist = tagliststring.split('\\n') + def shataglist = shatagliststring.split('\\n') LinkedHashMap> shatagmap = new LinkedHashMap>() for (shatag in shataglist) { From 462b17733fc0e81d423218022a15498d20656342 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 20:29:59 +0000 Subject: [PATCH 11/72] Build up a tag map --- versioncontrol/git.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 25221e0..711ae5f 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -41,7 +41,9 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL LinkedHashMap> shatagmap = new LinkedHashMap>() for (shatag in shataglist) { + echo "shatag: ${shatag}" def shatagfields = shatag.split(' ') + echo "shatagfields: ${shatagfields}" def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() shatagmap[sha].add(tag) From be599cbe86911a7f4b182d5e1939a13c93fa8f51 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 20:32:46 +0000 Subject: [PATCH 12/72] Build up a tag map --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 711ae5f..ecefc11 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -42,7 +42,7 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL LinkedHashMap> shatagmap = new LinkedHashMap>() for (shatag in shataglist) { echo "shatag: ${shatag}" - def shatagfields = shatag.split(' ') + def shatagfields = shatag.split('\\t') echo "shatagfields: ${shatagfields}" def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() From fb72031c9724489f234a6a949d87149bf879d832 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 20:39:10 +0000 Subject: [PATCH 13/72] Build up a tag map --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ecefc11..2ed4ac4 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -43,9 +43,9 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL for (shatag in shataglist) { echo "shatag: ${shatag}" def shatagfields = shatag.split('\\t') - echo "shatagfields: ${shatagfields}" def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() + echo "sha: ${sha} tag: ${tag}" shatagmap[sha].add(tag) } return shatagmap From 3dbdf85dfae6ac03537e006840d8a5e7d14892a8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 20:58:09 +0000 Subject: [PATCH 14/72] Build up a tag map --- versioncontrol/git.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 2ed4ac4..8afd5f3 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -41,12 +41,15 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL LinkedHashMap> shatagmap = new LinkedHashMap>() for (shatag in shataglist) { - echo "shatag: ${shatag}" def shatagfields = shatag.split('\\t') def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() echo "sha: ${sha} tag: ${tag}" - shatagmap[sha].add(tag) + if (shatagmap.get(sha) == null) { + shatagmap.put(sha, [tag]) + } else { + shatagmap.put(sha, shatagmap.get(sha).add(tag)) + } } return shatagmap } From 0ca7886778b3085b2596db3cc19b328db8aac4a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 21:08:09 +0000 Subject: [PATCH 15/72] Build up a tag map --- versioncontrol/git.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8afd5f3..3a8cc68 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -46,9 +46,13 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL def tag = shatagfields[1].trim() echo "sha: ${sha} tag: ${tag}" if (shatagmap.get(sha) == null) { - shatagmap.put(sha, [tag]) + def ArrayList taglist = new ArrayList() + taglist.add(tag) + shatagmap.put(sha, taglist) } else { - shatagmap.put(sha, shatagmap.get(sha).add(tag)) + def taglist = shatagmap.get(sha) + taglist.add(tag) + shatagmap.put(sha, taglist) } } return shatagmap From 4a46b8ffe062c14ad56a4c98ef2ebda0bd5f778f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 19 Feb 2022 21:09:29 +0000 Subject: [PATCH 16/72] Build up a tag map --- versioncontrol/git.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 3a8cc68..61378b5 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -35,7 +35,6 @@ ArrayList gettaglist(String workingSetRootPath) { } LinkedHashMap> getremoteshatagmap(String repositoryURL) { - ArrayList taglist = new ArrayList() def shatagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") def shataglist = shatagliststring.split('\\n') From 940a8d56bd327f0050cdc1e32f22532be7bbf082 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 22 Feb 2022 22:59:09 +0000 Subject: [PATCH 17/72] Start adding dependencies --- meta/folder.groovy | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6715145..7a92030 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -913,6 +913,14 @@ class MetaInfo extends GISJsonSerialised { String componentidentifier() { return productinfo.productname + "_" + productinfo.subcomponentname } + + String productvariant() { + if (variant == '') { + return productinfo.productname + } else { + return productinfo.productname + "_" + productinfo.variant + } + } String componentversionuniqueidentifier() { return componentidentifier() + "_" + buildstampinfo.buildstamp From 704e49969395635194b1fda7d3bf921305edebc7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 22 Feb 2022 23:03:10 +0000 Subject: [PATCH 18/72] Start adding dependencies --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7a92030..ac3d0dd 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -915,7 +915,7 @@ class MetaInfo extends GISJsonSerialised { } String productvariant() { - if (variant == '') { + if (!productinfo.variant) { return productinfo.productname } else { return productinfo.productname + "_" + productinfo.variant From 64882035b913280d8403a0bc34b8a5dd627ad495 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 24 Feb 2022 22:37:30 +0000 Subject: [PATCH 19/72] Supply release tag processing --- release/tag.groovy | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 release/tag.groovy diff --git a/release/tag.groovy b/release/tag.groovy new file mode 100644 index 0000000..be1944f --- /dev/null +++ b/release/tag.groovy @@ -0,0 +1,31 @@ + +def LinkedHashMap RELEASETAG_PREFIXES = [ + 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', + 'superseded' : 'Indicates a later release has superseded the release with this prefix', + 'erroneous' : 'The release was released unintentionally', + 'faulty' : 'The release was found to have a serious fault', + 'ignore' : 'The release should be ignored', + 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', +] + +def RELEASETAG_PREFIX_SEPERATOR = '_' + +def LinkedHashMap startswithspecialprefix(String releasetag) { + def boolean hasspecialprefix = false + def String releasetagprefix = '' + def String prefixfound = '' + def String tagwithoutprefix = releasetag + + def rtsplits = releasetag.split(RELEASETAG_PREFIX_SEPERATOR) + if (rtsplits.size() > 0) + prefixfound = rtsplits[0].toLowerCase() + hasspecialprefix = RELEASETAG_PREFIXES.containsKey(prefixfound) + tagwithoutprefix = rtsplits[1..].join(RELEASETAG_PREFIX_SEPERATOR) + } + return [ 'hasspecialprefix' : hasspecialprefix, + 'prefixfound' : prefixfound, + 'tagwithoutprefix' : tagwithoutprefix, + ] +} + +return this; From 80bc64e32eea7ef6c4528f0374dace9d19b4553b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 24 Feb 2022 23:16:06 +0000 Subject: [PATCH 20/72] Supply release tag processing --- release/tag.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index be1944f..2574fbc 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -16,11 +16,11 @@ def LinkedHashMap startswithspecialprefix(String releasetag) { def String prefixfound = '' def String tagwithoutprefix = releasetag - def rtsplits = releasetag.split(RELEASETAG_PREFIX_SEPERATOR) - if (rtsplits.size() > 0) - prefixfound = rtsplits[0].toLowerCase() + def firstseperatorindex = releasetag.indexOf(RELEASETAG_PREFIX_SEPERATOR) + if (firstseperatorindex > 0) + prefixfound = releasetag[0..(firstseperatorindex - 1)].toLowerCase() hasspecialprefix = RELEASETAG_PREFIXES.containsKey(prefixfound) - tagwithoutprefix = rtsplits[1..].join(RELEASETAG_PREFIX_SEPERATOR) + tagwithoutprefix = releasetag[(firstseperatorindex + 1)..-1] } return [ 'hasspecialprefix' : hasspecialprefix, 'prefixfound' : prefixfound, From 88cba3bc685a0fad146452b19990d5a23537e9c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 24 Feb 2022 23:18:11 +0000 Subject: [PATCH 21/72] Supply release tag processing --- release/tag.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/tag.groovy b/release/tag.groovy index 2574fbc..acfdb09 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -17,7 +17,7 @@ def LinkedHashMap startswithspecialprefix(String releasetag) { def String tagwithoutprefix = releasetag def firstseperatorindex = releasetag.indexOf(RELEASETAG_PREFIX_SEPERATOR) - if (firstseperatorindex > 0) + if (firstseperatorindex > 0) { prefixfound = releasetag[0..(firstseperatorindex - 1)].toLowerCase() hasspecialprefix = RELEASETAG_PREFIXES.containsKey(prefixfound) tagwithoutprefix = releasetag[(firstseperatorindex + 1)..-1] From ce33f59ea65a8008d1595080cbd0497c6d2a7c92 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 24 Feb 2022 23:42:04 +0000 Subject: [PATCH 22/72] Supply release tag processing --- release/tag.groovy | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index acfdb09..a6bbd28 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -16,11 +16,17 @@ def LinkedHashMap startswithspecialprefix(String releasetag) { def String prefixfound = '' def String tagwithoutprefix = releasetag + // Two forms of releasetag expected + // __ + // _ + def int seperatorcount = releasetag.split(RELEASETAG_PREFIX_SEPERATOR).size() - 1 def firstseperatorindex = releasetag.indexOf(RELEASETAG_PREFIX_SEPERATOR) - if (firstseperatorindex > 0) { - prefixfound = releasetag[0..(firstseperatorindex - 1)].toLowerCase() - hasspecialprefix = RELEASETAG_PREFIXES.containsKey(prefixfound) - tagwithoutprefix = releasetag[(firstseperatorindex + 1)..-1] + if (separatorcount >= 2) { + if (firstseperatorindex > 0) { + prefixfound = releasetag[0..(firstseperatorindex - 1)].toLowerCase() + hasspecialprefix = RELEASETAG_PREFIXES.containsKey(prefixfound) + tagwithoutprefix = releasetag[(firstseperatorindex + 1)..-1] + } } return [ 'hasspecialprefix' : hasspecialprefix, 'prefixfound' : prefixfound, From d2b8d1b7dc06197cda4783a6af6bdda19cd81815 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 24 Feb 2022 23:59:16 +0000 Subject: [PATCH 23/72] Supply release tag processing --- release/tag.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index a6bbd28..5238aa3 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -1,5 +1,5 @@ -def LinkedHashMap RELEASETAG_PREFIXES = [ +LinkedHashMap RELEASETAG_PREFIXES = [ 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', 'superseded' : 'Indicates a later release has superseded the release with this prefix', 'erroneous' : 'The release was released unintentionally', @@ -8,7 +8,7 @@ def LinkedHashMap RELEASETAG_PREFIXES = [ 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', ] -def RELEASETAG_PREFIX_SEPERATOR = '_' +RELEASETAG_PREFIX_SEPERATOR = '_' def LinkedHashMap startswithspecialprefix(String releasetag) { def boolean hasspecialprefix = false From 70d569c43bf09e6b613c5bebdd16a77e14baf8da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 00:01:09 +0000 Subject: [PATCH 24/72] Supply release tag processing --- release/tag.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/tag.groovy b/release/tag.groovy index 5238aa3..db6d59f 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -19,7 +19,7 @@ def LinkedHashMap startswithspecialprefix(String releasetag) { // Two forms of releasetag expected // __ // _ - def int seperatorcount = releasetag.split(RELEASETAG_PREFIX_SEPERATOR).size() - 1 + def int separatorcount = releasetag.split(RELEASETAG_PREFIX_SEPERATOR).size() - 1 def firstseperatorindex = releasetag.indexOf(RELEASETAG_PREFIX_SEPERATOR) if (separatorcount >= 2) { if (firstseperatorindex > 0) { From 89736f09fe503c0eecf873836b9db5e54527a551 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 00:06:23 +0000 Subject: [PATCH 25/72] Supply release tag processing --- release/tag.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/tag.groovy b/release/tag.groovy index db6d59f..63da06c 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -10,7 +10,7 @@ LinkedHashMap RELEASETAG_PREFIXES = [ RELEASETAG_PREFIX_SEPERATOR = '_' -def LinkedHashMap startswithspecialprefix(String releasetag) { +def startswithspecialprefix(String releasetag) { def boolean hasspecialprefix = false def String releasetagprefix = '' def String prefixfound = '' From 8b361de6538022600276a63eb5c3a2b18049a0a7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 00:12:00 +0000 Subject: [PATCH 26/72] Supply release tag processing --- release/tag.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/tag.groovy b/release/tag.groovy index 63da06c..87ac043 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -16,6 +16,7 @@ def startswithspecialprefix(String releasetag) { def String prefixfound = '' def String tagwithoutprefix = releasetag + echo "Release tag is: ${releasetag}" // Two forms of releasetag expected // __ // _ @@ -28,6 +29,7 @@ def startswithspecialprefix(String releasetag) { tagwithoutprefix = releasetag[(firstseperatorindex + 1)..-1] } } + echo "Results are: ${hasspecialprefix} ${prefixfound} ${tagwithoutprefix}" return [ 'hasspecialprefix' : hasspecialprefix, 'prefixfound' : prefixfound, 'tagwithoutprefix' : tagwithoutprefix, From af3c0415660d78b2c0cb1e772f27caf93a243d1f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 00:16:38 +0000 Subject: [PATCH 27/72] Supply release tag processing --- release/tag.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/tag.groovy b/release/tag.groovy index 87ac043..5d4d575 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -21,7 +21,9 @@ def startswithspecialprefix(String releasetag) { // __ // _ def int separatorcount = releasetag.split(RELEASETAG_PREFIX_SEPERATOR).size() - 1 + echo "separatorcount is: ${separatorcount}" def firstseperatorindex = releasetag.indexOf(RELEASETAG_PREFIX_SEPERATOR) + echo "firstseperatorindex is: ${firstseperatorindex}" if (separatorcount >= 2) { if (firstseperatorindex > 0) { prefixfound = releasetag[0..(firstseperatorindex - 1)].toLowerCase() From 44b1e13f31b55a229cb6679162d6259743ab17a7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 07:56:28 +0000 Subject: [PATCH 28/72] Supply release tag processing --- release/tag.groovy | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index 5d4d575..bb258ce 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -1,5 +1,5 @@ -LinkedHashMap RELEASETAG_PREFIXES = [ +def LinkedHashMap RELEASETAG_PREFIXES = [ 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', 'superseded' : 'Indicates a later release has superseded the release with this prefix', 'erroneous' : 'The release was released unintentionally', @@ -8,33 +8,29 @@ LinkedHashMap RELEASETAG_PREFIXES = [ 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', ] -RELEASETAG_PREFIX_SEPERATOR = '_' +def RELEASETAG_PREFIX_SEPERATOR = '_' def startswithspecialprefix(String releasetag) { def boolean hasspecialprefix = false def String releasetagprefix = '' def String prefixfound = '' + def String prefixmeaning = '' def String tagwithoutprefix = releasetag - echo "Release tag is: ${releasetag}" - // Two forms of releasetag expected - // __ - // _ - def int separatorcount = releasetag.split(RELEASETAG_PREFIX_SEPERATOR).size() - 1 - echo "separatorcount is: ${separatorcount}" - def firstseperatorindex = releasetag.indexOf(RELEASETAG_PREFIX_SEPERATOR) - echo "firstseperatorindex is: ${firstseperatorindex}" - if (separatorcount >= 2) { - if (firstseperatorindex > 0) { - prefixfound = releasetag[0..(firstseperatorindex - 1)].toLowerCase() - hasspecialprefix = RELEASETAG_PREFIXES.containsKey(prefixfound) - tagwithoutprefix = releasetag[(firstseperatorindex + 1)..-1] + RELEASETAG_PREFIXES.each { prefix, meaning -> + if (releasetag.toLowerCase().startsWith(prefix)) { + prefixfound = prefix + hasspecialprefix = true + prefixmeaning = meaning + tagwithoutprefix = releasetag[(prefix.size() + 1)..-1] + break } } echo "Results are: ${hasspecialprefix} ${prefixfound} ${tagwithoutprefix}" return [ 'hasspecialprefix' : hasspecialprefix, 'prefixfound' : prefixfound, 'tagwithoutprefix' : tagwithoutprefix, + 'meaning' : prefixmeaning, ] } From 4fbe1d9baae38b7062cac2ae4b0d804315733189 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 07:58:37 +0000 Subject: [PATCH 29/72] Supply release tag processing --- release/tag.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index bb258ce..b8d7b91 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -1,5 +1,5 @@ -def LinkedHashMap RELEASETAG_PREFIXES = [ +LinkedHashMap RELEASETAG_PREFIXES = [ 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', 'superseded' : 'Indicates a later release has superseded the release with this prefix', 'erroneous' : 'The release was released unintentionally', @@ -8,7 +8,7 @@ def LinkedHashMap RELEASETAG_PREFIXES = [ 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', ] -def RELEASETAG_PREFIX_SEPERATOR = '_' +RELEASETAG_PREFIX_SEPERATOR = '_' def startswithspecialprefix(String releasetag) { def boolean hasspecialprefix = false From b7c5a669f3b732d13d7e27900afce0cd8ce35024 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 08:01:08 +0000 Subject: [PATCH 30/72] Supply release tag processing --- release/tag.groovy | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index b8d7b91..19901d9 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -1,16 +1,16 @@ -LinkedHashMap RELEASETAG_PREFIXES = [ - 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', - 'superseded' : 'Indicates a later release has superseded the release with this prefix', - 'erroneous' : 'The release was released unintentionally', - 'faulty' : 'The release was found to have a serious fault', - 'ignore' : 'The release should be ignored', - 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', -] - -RELEASETAG_PREFIX_SEPERATOR = '_' - def startswithspecialprefix(String releasetag) { + def LinkedHashMap RELEASETAG_PREFIXES = [ + 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', + 'superseded' : 'Indicates a later release has superseded the release with this prefix', + 'erroneous' : 'The release was released unintentionally', + 'faulty' : 'The release was found to have a serious fault', + 'ignore' : 'The release should be ignored', + 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', + ] + + def RELEASETAG_PREFIX_SEPERATOR = '_' + def boolean hasspecialprefix = false def String releasetagprefix = '' def String prefixfound = '' From af3e22db939a8332117ad3cc3f98d3d34c00dbd3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 08:04:50 +0000 Subject: [PATCH 31/72] Supply release tag processing --- release/tag.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index 19901d9..b17a14e 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -1,6 +1,6 @@ def startswithspecialprefix(String releasetag) { - def LinkedHashMap RELEASETAG_PREFIXES = [ + LinkedHashMap RELEASETAG_PREFIXES = [ 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', 'superseded' : 'Indicates a later release has superseded the release with this prefix', 'erroneous' : 'The release was released unintentionally', @@ -9,7 +9,7 @@ def startswithspecialprefix(String releasetag) { 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', ] - def RELEASETAG_PREFIX_SEPERATOR = '_' + RELEASETAG_PREFIX_SEPERATOR = '_' def boolean hasspecialprefix = false def String releasetagprefix = '' From 01ce78080191d0a6927ba45e3f032124ea088918 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 08:11:27 +0000 Subject: [PATCH 32/72] Supply release tag processing --- release/tag.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/tag.groovy b/release/tag.groovy index b17a14e..0214eca 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -26,7 +26,6 @@ def startswithspecialprefix(String releasetag) { break } } - echo "Results are: ${hasspecialprefix} ${prefixfound} ${tagwithoutprefix}" return [ 'hasspecialprefix' : hasspecialprefix, 'prefixfound' : prefixfound, 'tagwithoutprefix' : tagwithoutprefix, From 32a3aeda0fce8162a58e3477ee63c74bb93e3645 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 08:17:53 +0000 Subject: [PATCH 33/72] Supply release tag processing --- release/tag.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/release/tag.groovy b/release/tag.groovy index 0214eca..90d6bbd 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -26,11 +26,13 @@ def startswithspecialprefix(String releasetag) { break } } - return [ 'hasspecialprefix' : hasspecialprefix, - 'prefixfound' : prefixfound, - 'tagwithoutprefix' : tagwithoutprefix, - 'meaning' : prefixmeaning, - ] + + LinkedHashMap results = [ 'hasspecialprefix' : hasspecialprefix, + 'prefixfound' : prefixfound, + 'tagwithoutprefix' : tagwithoutprefix, + 'meaning' : prefixmeaning, + ] + return results } return this; From a576d6403fe00c78a14f433faf76a8c4046e6e03 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 08:38:54 +0000 Subject: [PATCH 34/72] Script reorg --- product/info.groovy | 185 ++++++++++++++++++++++++++++++++++++++++++++ release/tag.groovy | 16 ++-- 2 files changed, 193 insertions(+), 8 deletions(-) create mode 100644 product/info.groovy diff --git a/product/info.groovy b/product/info.groovy new file mode 100644 index 0000000..e0bbede --- /dev/null +++ b/product/info.groovy @@ -0,0 +1,185 @@ + +def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") +def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +def rt = load("${autobuilder.commonscripts()}/release/tag.groovy") + +class ProductDependency { + private script + private String productvariant + private String releaserepo + private String releasetag + private String releasestate + + def ProductDependency(script, String productvariant, String releasetag, String releaserepo) { + this.script = script + this.productvariant = productvariant + this.releaserepo = releaserepo + this.releasetag = releasetag + this.releasestate = '' + } + + def setreleasestate(rt) { + def LinkedHashMap results = rt.startswithspecialprefix(releasetag) + if (results.hasspecialprefix) { + releasetag = results.tagwithoutprefix + releasestate = results.prefixfound + } else { + releasetag = results.tagwithoutprefix + releasestate = '' + } + } +} + +class ProductComponent { + private String componentname + private String releaserepo + private String releasetag + + def ProductComponent(String componentname, String releasetag, String releaserepo) { + this.componentname = componentname + this.releaserepo = releaserepo + this.releasetag = releasetag + } +} + +class ProductRelease { + private String sha + private String builddate + private ArrayList tags + private ArrayList productdependencies + private ArrayList productcomponents + + def ProductRelease(String sha, String builddate, ArrayList tags) { + this.sha = sha + this.builddate = builddate + this.tags = tags + this.productdependencies = new ArrayList() + this.productcomponents = new ArrayList() + } + + def addProductDependency(ProductDependency dependency) { + productdependencies.add(dependency) + } + + def addProductComponent(ProductComponent component) { + productcomponents.add(component) + } +} + +class ProductParsed { + enum State { + UNPARSED, + PARSED, + } + private State scanstate + private String productvariant + private ArrayList releases + + def ProductParsed(String productvariant) { + scanstate = State.UNPARSED + this.productvariant = productvariant + this.releases = new ArrayList() + } + + def markparsed() { + scanstate = State.PARSED + } + + def boolean parsed() { + return (scanstate == State.PARSED) + } + + def addProductRelease(ProductRelease productrelease) { + releases.add(productrelease) + } +} + +def LinkedHashMap products = new LinkedHashMap() + +def addProductEntry(LinkedHashMap products, + String productvariant, + String releaserepo) { + def productparsed + if (products.get(releaserepo) == null) { // Use the first entry name only + productparsed = new ProductParsed(productvariant) + products.put(releaserepo, productparsed) + } else { + productparsed = products.get(releaserepo) + } + return productparsed +} + +def parseproductdependencies(rt, + productreleasemeta, + String productreleasefolder, + String containingproductvariant, + ProductRelease productrelease, + LinkedHashMap products) { + def dependentmeta = productreleasemeta.getassociated(productreleasefolder) + + for (meta in dependentmeta) { + if (meta.productinfo.subcomponentname != 'WHOLE') { + def pc = new ProductComponent(meta.productinfo.subcomponentname, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductComponent(pc) + } else { + if (meta.productvariant() != containingproductvariant) { + def pd = new ProductDependency(meta.productvariant(), + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + pd.setreleasestate(rt) + addProductEntry(products, + meta.productvariant(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } + } + } +} + +def parseproductrepo(String productrepo, + String productreleasefolder) { + def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) + def String productvariant + def ProductParsed pp + + def ArrayList productreleases = new ArrayList() + shatagmap.each { sha, tags -> + def importoptions = importer.getNewImportOptions() + importoptions.releaserepo(productrepo, sha) + importoptions.setfilter('meta/**') + def productreleasemeta = importer.artifactimport(meta, importoptions, productreleasefolder) + + if (!productvariant) { + productvariant = productreleasemeta.productvariant() + pp = addProductEntry(productvariant, productrepo) + } + + def productrelease = new ProductRelease(sha, productreleasemeta.jenkinsbuildinfo.builddate, tags) + parseproductdependencies(productreleasemeta, productreleasefolder, productvariant, productrelease, products) + pp.addProductRelease(productrelease) + } + pp.markparsed() +} + +def iterateinitialproductrepos(productcollection, String productreleasefolder) { + for (productrepo in productcollection.ProductReleaseRepos) { + echo "Querying ${productrepo}" + parseproductrepo(productrepo, + productreleasefolder) + } + while( true ) { + boolean allparsed = true + products.each { releaserepo, productparsed -> + if (!productparsed.parsed()) { + allparsed = false + parseproductrepo(releaserepo, + productreleasefolder) + } + } + if (allparsed) break + } +} + +return this \ No newline at end of file diff --git a/release/tag.groovy b/release/tag.groovy index 90d6bbd..cb9dd6c 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -1,6 +1,6 @@ def startswithspecialprefix(String releasetag) { - LinkedHashMap RELEASETAG_PREFIXES = [ + def RELEASETAG_PREFIXES = [ 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', 'superseded' : 'Indicates a later release has superseded the release with this prefix', 'erroneous' : 'The release was released unintentionally', @@ -9,7 +9,7 @@ def startswithspecialprefix(String releasetag) { 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', ] - RELEASETAG_PREFIX_SEPERATOR = '_' + def RELEASETAG_PREFIX_SEPERATOR = '_' def boolean hasspecialprefix = false def String releasetagprefix = '' @@ -27,12 +27,12 @@ def startswithspecialprefix(String releasetag) { } } - LinkedHashMap results = [ 'hasspecialprefix' : hasspecialprefix, - 'prefixfound' : prefixfound, - 'tagwithoutprefix' : tagwithoutprefix, - 'meaning' : prefixmeaning, - ] + def results = [ 'hasspecialprefix' : hasspecialprefix, + 'prefixfound' : prefixfound, + 'tagwithoutprefix' : tagwithoutprefix, + 'meaning' : prefixmeaning, + ] return results } -return this; +return this From 57eeaf4bc7a26eaaa25c111beb0a064e8864195c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 22:43:55 +0000 Subject: [PATCH 35/72] Comment out code to debug later --- product/info.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/product/info.groovy b/product/info.groovy index e0bbede..cecba6c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -3,6 +3,7 @@ def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def rt = load("${autobuilder.commonscripts()}/release/tag.groovy") +/* class ProductDependency { private script private String productvariant @@ -181,5 +182,6 @@ def iterateinitialproductrepos(productcollection, String productreleasefolder) { if (allparsed) break } } +*/ return this \ No newline at end of file From e6c6fcf5e433ea90879a17515c500e09268ec6be Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 22:53:53 +0000 Subject: [PATCH 36/72] Debugging --- product/info.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index cecba6c..321e8d0 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -163,10 +163,12 @@ def parseproductrepo(String productrepo, } pp.markparsed() } +*/ def iterateinitialproductrepos(productcollection, String productreleasefolder) { for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" +/* parseproductrepo(productrepo, productreleasefolder) } @@ -180,8 +182,8 @@ def iterateinitialproductrepos(productcollection, String productreleasefolder) { } } if (allparsed) break + */ } } -*/ return this \ No newline at end of file From b1bc5408883b6d301d59b4d5f8285bd2fb02333c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 23:03:47 +0000 Subject: [PATCH 37/72] Debugging --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 321e8d0..b08feea 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -165,7 +165,7 @@ def parseproductrepo(String productrepo, } */ -def iterateinitialproductrepos(productcollection, String productreleasefolder) { +def iterateinitialproductrepos(JSONObject productcollection, String productreleasefolder) { for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" /* From ed17de0309b00db351be542f07e2ad8ea83a3d3e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 23:07:02 +0000 Subject: [PATCH 38/72] Debugging --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index b08feea..6542b6b 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -165,7 +165,7 @@ def parseproductrepo(String productrepo, } */ -def iterateinitialproductrepos(JSONObject productcollection, String productreleasefolder) { +def iterateinitialproductrepos(JSONArray productcollection, String productreleasefolder) { for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" /* From 5771490114d5d7e92f76f088d28896a35d5292d3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Feb 2022 23:13:42 +0000 Subject: [PATCH 39/72] Debugging --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 6542b6b..321e8d0 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -165,7 +165,7 @@ def parseproductrepo(String productrepo, } */ -def iterateinitialproductrepos(JSONArray productcollection, String productreleasefolder) { +def iterateinitialproductrepos(productcollection, String productreleasefolder) { for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" /* From 68cbc00f3c28936169f2b99ede6c8b32719c92cd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:12:12 +0000 Subject: [PATCH 40/72] Debugging script --- product/info.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 321e8d0..6cd70ad 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -3,7 +3,6 @@ def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def rt = load("${autobuilder.commonscripts()}/release/tag.groovy") -/* class ProductDependency { private script private String productvariant @@ -97,6 +96,7 @@ class ProductParsed { def LinkedHashMap products = new LinkedHashMap() +/* def addProductEntry(LinkedHashMap products, String productvariant, String releaserepo) { @@ -182,8 +182,8 @@ def iterateinitialproductrepos(productcollection, String productreleasefolder) { } } if (allparsed) break - */ } + */ } return this \ No newline at end of file From 113186532c51860cbf9f5aee2edee90f89854cca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:15:32 +0000 Subject: [PATCH 41/72] Debugging script --- product/info.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/product/info.groovy b/product/info.groovy index 6cd70ad..8cbdb58 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -171,7 +171,9 @@ def iterateinitialproductrepos(productcollection, String productreleasefolder) { /* parseproductrepo(productrepo, productreleasefolder) +*/ } + /* while( true ) { boolean allparsed = true products.each { releaserepo, productparsed -> From b187d23be32cc0dddace926d28b34f8cc5ce8324 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:25:54 +0000 Subject: [PATCH 42/72] Debugging script --- product/info.groovy | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 8cbdb58..6fccece 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -165,10 +165,18 @@ def parseproductrepo(String productrepo, } */ -def iterateinitialproductrepos(productcollection, String productreleasefolder) { +def iterateinitialproductrepos(script, productcollection, String productreleasefolder) { for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" -/* + + def String productvariant = 'HMBFDHV_sg1024' + def String releasetag = '1.2.3.4_54' + def releaserepo = 'git@hydra/blah.git' + def pd = new ProductDependency(script, + productvariant, + releasetag, + releaserepo) + /* parseproductrepo(productrepo, productreleasefolder) */ From 1d3f18418fa546184926b9c9378d6f7388cf7a53 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:29:41 +0000 Subject: [PATCH 43/72] Debugging script --- product/info.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 6fccece..3087aac 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -165,10 +165,12 @@ def parseproductrepo(String productrepo, } */ -def iterateinitialproductrepos(script, productcollection, String productreleasefolder) { - for (productrepo in productcollection.ProductReleaseRepos) { +def iterateinitialproductrepos(script, + productcollection, + String productreleasefolder) { +/* for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" - +*/ def String productvariant = 'HMBFDHV_sg1024' def String releasetag = '1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' @@ -179,7 +181,6 @@ def iterateinitialproductrepos(script, productcollection, String productreleasef /* parseproductrepo(productrepo, productreleasefolder) -*/ } /* while( true ) { From ac5e7cdbbffac01b43e21e19cc0a64076111cfdb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:34:48 +0000 Subject: [PATCH 44/72] Debugging script --- product/info.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/product/info.groovy b/product/info.groovy index 3087aac..b479bc5 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -178,6 +178,7 @@ def iterateinitialproductrepos(script, productvariant, releasetag, releaserepo) + script.echo("pd is: ${pd}") /* parseproductrepo(productrepo, productreleasefolder) From 38b562b1a6909f64c7f95df2897da089ad8acee3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:38:19 +0000 Subject: [PATCH 45/72] Debugging script --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index b479bc5..f98694a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -178,7 +178,7 @@ def iterateinitialproductrepos(script, productvariant, releasetag, releaserepo) - script.echo("pd is: ${pd}") + script.echo("pd is: ${pd.dump()}") /* parseproductrepo(productrepo, productreleasefolder) From 48d7b0fc338e075d29bbb8104bebdb4711c83601 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:46:42 +0000 Subject: [PATCH 46/72] Debugging script --- product/info.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index f98694a..ea0d5b1 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -18,8 +18,9 @@ class ProductDependency { this.releasestate = '' } - def setreleasestate(rt) { + def setreleasestate() { def LinkedHashMap results = rt.startswithspecialprefix(releasetag) + script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix releasestate = results.prefixfound @@ -179,6 +180,8 @@ def iterateinitialproductrepos(script, releasetag, releaserepo) script.echo("pd is: ${pd.dump()}") + + pd.setreleasestate() /* parseproductrepo(productrepo, productreleasefolder) From 04e947cbfdef1dfe182c4c2e98789e7d732726e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:51:28 +0000 Subject: [PATCH 47/72] Debugging script --- product/info.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index ea0d5b1..8283615 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,13 +5,19 @@ def rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { private script + private rt private String productvariant private String releaserepo private String releasetag private String releasestate - def ProductDependency(script, String productvariant, String releasetag, String releaserepo) { + def ProductDependency(script, + rt, + String productvariant, + String releasetag, + String releaserepo) { this.script = script + this.rt = rt this.productvariant = productvariant this.releaserepo = releaserepo this.releasetag = releasetag @@ -176,6 +182,7 @@ def iterateinitialproductrepos(script, def String releasetag = '1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' def pd = new ProductDependency(script, + rt, productvariant, releasetag, releaserepo) From f34fb2e2fafd775356467fd099f0a5de2f06eacc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:55:44 +0000 Subject: [PATCH 48/72] Debugging script --- product/info.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 8283615..e844c59 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,19 +5,19 @@ def rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { private script - private rt + private rtscript private String productvariant private String releaserepo private String releasetag private String releasestate def ProductDependency(script, - rt, + rtscript, String productvariant, String releasetag, String releaserepo) { this.script = script - this.rt = rt + this.rtscript = rtscript this.productvariant = productvariant this.releaserepo = releaserepo this.releasetag = releasetag @@ -25,7 +25,7 @@ class ProductDependency { } def setreleasestate() { - def LinkedHashMap results = rt.startswithspecialprefix(releasetag) + def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix From d418e5ab107f830c6d8517bad2422f7cae5d3428 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 14:58:45 +0000 Subject: [PATCH 49/72] Debugging script --- product/info.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index e844c59..d777bbf 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,7 +1,7 @@ -def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") -def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def rt = load("${autobuilder.commonscripts()}/release/tag.groovy") +importer = load("${autobuilder.commonscripts()}/import/importer.groovy") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { private script From fa199eb5ffcd5da5721bc22213e828f286a7dff6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:03:09 +0000 Subject: [PATCH 50/72] Debugging script --- product/info.groovy | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index d777bbf..8e2cefb 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,19 +5,16 @@ rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { private script - private rtscript private String productvariant private String releaserepo private String releasetag private String releasestate def ProductDependency(script, - rtscript, String productvariant, String releasetag, String releaserepo) { this.script = script - this.rtscript = rtscript this.productvariant = productvariant this.releaserepo = releaserepo this.releasetag = releasetag @@ -25,7 +22,7 @@ class ProductDependency { } def setreleasestate() { - def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) + def LinkedHashMap results = rt.startswithspecialprefix(releasetag) script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix @@ -182,7 +179,6 @@ def iterateinitialproductrepos(script, def String releasetag = '1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' def pd = new ProductDependency(script, - rt, productvariant, releasetag, releaserepo) From c2faf8abc6bb620a6b2c96a7dc44a04d6d24810e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:14:04 +0000 Subject: [PATCH 51/72] Debugging script --- product/info.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 8e2cefb..d777bbf 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,16 +5,19 @@ rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { private script + private rtscript private String productvariant private String releaserepo private String releasetag private String releasestate def ProductDependency(script, + rtscript, String productvariant, String releasetag, String releaserepo) { this.script = script + this.rtscript = rtscript this.productvariant = productvariant this.releaserepo = releaserepo this.releasetag = releasetag @@ -22,7 +25,7 @@ class ProductDependency { } def setreleasestate() { - def LinkedHashMap results = rt.startswithspecialprefix(releasetag) + def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix @@ -179,6 +182,7 @@ def iterateinitialproductrepos(script, def String releasetag = '1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' def pd = new ProductDependency(script, + rt, productvariant, releasetag, releaserepo) From b5e289411cf028428ade167ebe1eae88c5ceb6c0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:17:14 +0000 Subject: [PATCH 52/72] Debugging script --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index d777bbf..18bcef3 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -179,7 +179,7 @@ def iterateinitialproductrepos(script, echo "Querying ${productrepo}" */ def String productvariant = 'HMBFDHV_sg1024' - def String releasetag = '1.2.3.4_54' + def String releasetag = 'inmainline_1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' def pd = new ProductDependency(script, rt, From f2e3e526eddf5ebe8c6826d97a631f993a0402ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:30:24 +0000 Subject: [PATCH 53/72] Debugging script --- release/tag.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/tag.groovy b/release/tag.groovy index cb9dd6c..6155e7f 100644 --- a/release/tag.groovy +++ b/release/tag.groovy @@ -17,7 +17,8 @@ def startswithspecialprefix(String releasetag) { def String prefixmeaning = '' def String tagwithoutprefix = releasetag - RELEASETAG_PREFIXES.each { prefix, meaning -> + for (def prefix in RELEASETAG_PREFIXES.keySet()) { + def meaning = RELEASETAG_PREFIXES.get(prefix) if (releasetag.toLowerCase().startsWith(prefix)) { prefixfound = prefix hasspecialprefix = true From defc807e97935826dd31e30557b1e3d5b140d963 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:34:24 +0000 Subject: [PATCH 54/72] Debugging script --- product/info.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 18bcef3..23f898d 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -186,10 +186,10 @@ def iterateinitialproductrepos(script, productvariant, releasetag, releaserepo) + pd.setreleasestate() script.echo("pd is: ${pd.dump()}") - pd.setreleasestate() - /* + /* parseproductrepo(productrepo, productreleasefolder) } From 35d3862b258b00125f4978a295bb90be383b0ffd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:40:36 +0000 Subject: [PATCH 55/72] Debugging script --- product/info.groovy | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 23f898d..c598bbf 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,6 @@ rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { private script - private rtscript private String productvariant private String releaserepo private String releasetag @@ -17,22 +16,15 @@ class ProductDependency { String releasetag, String releaserepo) { this.script = script - this.rtscript = rtscript this.productvariant = productvariant this.releaserepo = releaserepo - this.releasetag = releasetag - this.releasestate = '' - } - - def setreleasestate() { def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) - script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { - releasetag = results.tagwithoutprefix - releasestate = results.prefixfound + this.releasetag = results.tagwithoutprefix + this.releasestate = results.prefixfound } else { - releasetag = results.tagwithoutprefix - releasestate = '' + this.releasetag = results.tagwithoutprefix + this.releasestate = '' } } } @@ -186,7 +178,6 @@ def iterateinitialproductrepos(script, productvariant, releasetag, releaserepo) - pd.setreleasestate() script.echo("pd is: ${pd.dump()}") /* From 2939eaf4b2753df7bb41961ba0d7a1e0453e750b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:50:43 +0000 Subject: [PATCH 56/72] Debugging script --- product/info.groovy | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index c598bbf..657b027 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -4,27 +4,28 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { - private script private String productvariant private String releaserepo private String releasetag private String releasestate - def ProductDependency(script, - rtscript, - String productvariant, + def ProductDependency(String productvariant, String releasetag, String releaserepo) { - this.script = script this.productvariant = productvariant this.releaserepo = releaserepo + this.releasetag = releasetag + this.releasestate = '' + } + + def setreleasestate(rtscript) { def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) if (results.hasspecialprefix) { - this.releasetag = results.tagwithoutprefix - this.releasestate = results.prefixfound + releasetag = results.tagwithoutprefix + releasestate = results.prefixfound } else { - this.releasetag = results.tagwithoutprefix - this.releasestate = '' + releasetag = results.tagwithoutprefix + releasestate = '' } } } @@ -173,12 +174,11 @@ def iterateinitialproductrepos(script, def String productvariant = 'HMBFDHV_sg1024' def String releasetag = 'inmainline_1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' - def pd = new ProductDependency(script, - rt, - productvariant, + def pd = new ProductDependency(productvariant, releasetag, releaserepo) - script.echo("pd is: ${pd.dump()}") + pd.setreleasestate(rt) + script.echo("pd is: ${pd.dump()}") /* parseproductrepo(productrepo, From 490b37bd7a2e339ed739e7af90a878d48a0c2f6d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 15:57:37 +0000 Subject: [PATCH 57/72] Debugging script --- product/info.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 657b027..6f5ec2b 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -21,11 +21,11 @@ class ProductDependency { def setreleasestate(rtscript) { def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) if (results.hasspecialprefix) { - releasetag = results.tagwithoutprefix - releasestate = results.prefixfound + this.releasetag = results.tagwithoutprefix + this.releasestate = results.prefixfound } else { - releasetag = results.tagwithoutprefix - releasestate = '' + this.releasetag = results.tagwithoutprefix + this.releasestate = '' } } } From 30ac58da78d6bd5fce2d89a93fe66443488f1122 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:00:49 +0000 Subject: [PATCH 58/72] Debugging script --- product/info.groovy | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 6f5ec2b..afc8da4 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -4,28 +4,35 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") rt = load("${autobuilder.commonscripts()}/release/tag.groovy") class ProductDependency { + private script + private rtscript private String productvariant private String releaserepo private String releasetag private String releasestate - def ProductDependency(String productvariant, + def ProductDependency(script, + rtscript, + String productvariant, String releasetag, String releaserepo) { + this.script = script + this.rtscript = rtscript this.productvariant = productvariant this.releaserepo = releaserepo this.releasetag = releasetag this.releasestate = '' } - def setreleasestate(rtscript) { + def setreleasestate() { def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) + script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { - this.releasetag = results.tagwithoutprefix - this.releasestate = results.prefixfound + releasetag = results.tagwithoutprefix + releasestate = results.prefixfound } else { - this.releasetag = results.tagwithoutprefix - this.releasestate = '' + releasetag = results.tagwithoutprefix + releasestate = '' } } } @@ -174,11 +181,13 @@ def iterateinitialproductrepos(script, def String productvariant = 'HMBFDHV_sg1024' def String releasetag = 'inmainline_1.2.3.4_54' def releaserepo = 'git@hydra/blah.git' - def pd = new ProductDependency(productvariant, + def pd = new ProductDependency(script, + rt, + productvariant, releasetag, releaserepo) - pd.setreleasestate(rt) - script.echo("pd is: ${pd.dump()}") + pd.setreleasestate() + script.echo("pd is: ${pd.dump()}") /* parseproductrepo(productrepo, From d2a22db3aba3403e8b9c140683d132c831c583c0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:15:25 +0000 Subject: [PATCH 59/72] Debugging script --- product/info.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index afc8da4..7058c9d 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -8,24 +8,26 @@ class ProductDependency { private rtscript private String productvariant private String releaserepo + private String presplitreleasetag private String releasetag private String releasestate def ProductDependency(script, rtscript, String productvariant, - String releasetag, + String presplitreleasetag, String releaserepo) { this.script = script this.rtscript = rtscript this.productvariant = productvariant this.releaserepo = releaserepo - this.releasetag = releasetag + this.presplitreleasetag = presplitreleasetag + this.releasetag = '' this.releasestate = '' } def setreleasestate() { - def LinkedHashMap results = rtscript.startswithspecialprefix(releasetag) + def LinkedHashMap results = rtscript.startswithspecialprefix(presplitreleasetag) script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix From 01d91a7ec10461804df652b6fb6e774e36f4b83b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:24:19 +0000 Subject: [PATCH 60/72] Debugging script --- product/info.groovy | 42 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 7058c9d..be95978 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -105,7 +105,6 @@ class ProductParsed { def LinkedHashMap products = new LinkedHashMap() -/* def addProductEntry(LinkedHashMap products, String productvariant, String releaserepo) { @@ -119,8 +118,7 @@ def addProductEntry(LinkedHashMap products, return productparsed } -def parseproductdependencies(rt, - productreleasemeta, +def parseproductdependencies(productreleasemeta, String productreleasefolder, String containingproductvariant, ProductRelease productrelease, @@ -135,10 +133,12 @@ def parseproductdependencies(rt, productrelease.addProductComponent(pc) } else { if (meta.productvariant() != containingproductvariant) { - def pd = new ProductDependency(meta.productvariant(), + def pd = new ProductDependency(this, + rt, + meta.productvariant(), meta.releasetag(), meta.releaseinfo.releaserepoURL) - pd.setreleasestate(rt) + pd.setreleasestate() addProductEntry(products, meta.productvariant(), meta.releaseinfo.releaserepoURL) @@ -159,43 +159,36 @@ def parseproductrepo(String productrepo, def importoptions = importer.getNewImportOptions() importoptions.releaserepo(productrepo, sha) importoptions.setfilter('meta/**') - def productreleasemeta = importer.artifactimport(meta, importoptions, productreleasefolder) + def productreleasemeta = importer.artifactimport(meta, + importoptions, + productreleasefolder) if (!productvariant) { productvariant = productreleasemeta.productvariant() pp = addProductEntry(productvariant, productrepo) } - def productrelease = new ProductRelease(sha, productreleasemeta.jenkinsbuildinfo.builddate, tags) - parseproductdependencies(productreleasemeta, productreleasefolder, productvariant, productrelease, products) + def productrelease = new ProductRelease(sha, + productreleasemeta.jenkinsbuildinfo.builddate, + tags) + parseproductdependencies(productreleasemeta, + productreleasefolder, + productvariant, + productrelease, + products) pp.addProductRelease(productrelease) } pp.markparsed() } -*/ def iterateinitialproductrepos(script, productcollection, String productreleasefolder) { -/* for (productrepo in productcollection.ProductReleaseRepos) { + for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" -*/ - def String productvariant = 'HMBFDHV_sg1024' - def String releasetag = 'inmainline_1.2.3.4_54' - def releaserepo = 'git@hydra/blah.git' - def pd = new ProductDependency(script, - rt, - productvariant, - releasetag, - releaserepo) - pd.setreleasestate() - script.echo("pd is: ${pd.dump()}") - - /* parseproductrepo(productrepo, productreleasefolder) } - /* while( true ) { boolean allparsed = true products.each { releaserepo, productparsed -> @@ -207,7 +200,6 @@ def iterateinitialproductrepos(script, } if (allparsed) break } - */ } return this \ No newline at end of file From d5a178bee9ae68f7078496b878da33325546bf1b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:28:15 +0000 Subject: [PATCH 61/72] Debugging script --- product/info.groovy | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index be95978..18c7ac2 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -103,10 +103,9 @@ class ProductParsed { } } -def LinkedHashMap products = new LinkedHashMap() +LinkedHashMap products = new LinkedHashMap() -def addProductEntry(LinkedHashMap products, - String productvariant, +def addProductEntry(String productvariant, String releaserepo) { def productparsed if (products.get(releaserepo) == null) { // Use the first entry name only @@ -121,8 +120,7 @@ def addProductEntry(LinkedHashMap products, def parseproductdependencies(productreleasemeta, String productreleasefolder, String containingproductvariant, - ProductRelease productrelease, - LinkedHashMap products) { + ProductRelease productrelease) { def dependentmeta = productreleasemeta.getassociated(productreleasefolder) for (meta in dependentmeta) { @@ -165,7 +163,8 @@ def parseproductrepo(String productrepo, if (!productvariant) { productvariant = productreleasemeta.productvariant() - pp = addProductEntry(productvariant, productrepo) + pp = addProductEntry(products, + productvariant, productrepo) } def productrelease = new ProductRelease(sha, @@ -174,8 +173,7 @@ def parseproductrepo(String productrepo, parseproductdependencies(productreleasemeta, productreleasefolder, productvariant, - productrelease, - products) + productrelease) pp.addProductRelease(productrelease) } pp.markparsed() From 0ad763cd295aa78df32f1b790374576846b16110 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:31:53 +0000 Subject: [PATCH 62/72] Debugging script --- product/info.groovy | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 18c7ac2..433003c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -105,7 +105,8 @@ class ProductParsed { LinkedHashMap products = new LinkedHashMap() -def addProductEntry(String productvariant, +def addProductEntry(LinkedHashMap products, + String productvariant, String releaserepo) { def productparsed if (products.get(releaserepo) == null) { // Use the first entry name only @@ -117,7 +118,8 @@ def addProductEntry(String productvariant, return productparsed } -def parseproductdependencies(productreleasemeta, +def parseproductdependencies(LinkedHashMap products, + productreleasemeta, String productreleasefolder, String containingproductvariant, ProductRelease productrelease) { @@ -170,7 +172,8 @@ def parseproductrepo(String productrepo, def productrelease = new ProductRelease(sha, productreleasemeta.jenkinsbuildinfo.builddate, tags) - parseproductdependencies(productreleasemeta, + parseproductdependencies(products, + productreleasemeta, productreleasefolder, productvariant, productrelease) From 2e5b6b212141b33eab6c407ea86827fad5e5f589 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:34:56 +0000 Subject: [PATCH 63/72] Debugging script --- product/info.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 433003c..3ad7968 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -148,7 +148,8 @@ def parseproductdependencies(LinkedHashMap products, } } -def parseproductrepo(String productrepo, +def parseproductrepo(LinkedHashMap products, + String productrepo, String productreleasefolder) { def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) def String productvariant @@ -182,12 +183,12 @@ def parseproductrepo(String productrepo, pp.markparsed() } -def iterateinitialproductrepos(script, - productcollection, +def iterateinitialproductrepos(productcollection, String productreleasefolder) { for (productrepo in productcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" - parseproductrepo(productrepo, + parseproductrepo(products, + productrepo, productreleasefolder) } while( true ) { @@ -195,7 +196,8 @@ def iterateinitialproductrepos(script, products.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false - parseproductrepo(releaserepo, + parseproductrepo(products, + releaserepo, productreleasefolder) } } From c993ecc75bdc11539ee197dd96f9413735ce4f0e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:40:31 +0000 Subject: [PATCH 64/72] Debugging script --- product/info.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 3ad7968..e004b87 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -103,7 +103,7 @@ class ProductParsed { } } -LinkedHashMap products = new LinkedHashMap() +LinkedHashMap productscollection = new LinkedHashMap() def addProductEntry(LinkedHashMap products, String productvariant, @@ -183,11 +183,11 @@ def parseproductrepo(LinkedHashMap products, pp.markparsed() } -def iterateinitialproductrepos(productcollection, +def iterateinitialproductrepos(initialproductcollection, String productreleasefolder) { - for (productrepo in productcollection.ProductReleaseRepos) { + for (productrepo in initialproductcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" - parseproductrepo(products, + parseproductrepo(productscollection, productrepo, productreleasefolder) } @@ -196,7 +196,7 @@ def iterateinitialproductrepos(productcollection, products.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false - parseproductrepo(products, + parseproductrepo(productscollection, releaserepo, productreleasefolder) } From caf71e38cedc351cb29d0353f0c3eeddc53cf632 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 16:47:37 +0000 Subject: [PATCH 65/72] Debugging script --- product/info.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index e004b87..d71177d 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -103,8 +103,6 @@ class ProductParsed { } } -LinkedHashMap productscollection = new LinkedHashMap() - def addProductEntry(LinkedHashMap products, String productvariant, String releaserepo) { @@ -184,10 +182,11 @@ def parseproductrepo(LinkedHashMap products, } def iterateinitialproductrepos(initialproductcollection, + LinkedHashMap productcollection, String productreleasefolder) { for (productrepo in initialproductcollection.ProductReleaseRepos) { echo "Querying ${productrepo}" - parseproductrepo(productscollection, + parseproductrepo(productcollection, productrepo, productreleasefolder) } @@ -196,7 +195,7 @@ def iterateinitialproductrepos(initialproductcollection, products.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false - parseproductrepo(productscollection, + parseproductrepo(productcollection, releaserepo, productreleasefolder) } From d93f9429b1f3f245a12279854ea6d8e4e4a8db4c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 17:02:17 +0000 Subject: [PATCH 66/72] Debugging script --- product/info.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/product/info.groovy b/product/info.groovy index d71177d..07a0b86 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -204,4 +204,9 @@ def iterateinitialproductrepos(initialproductcollection, } } +def LinkedHashMap getNewProductCollection() { + return new LinkedHashMap() +} + + return this \ No newline at end of file From 3c4de238e40c6dcc004e0ccbc3bf318577b6f418 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 17:04:37 +0000 Subject: [PATCH 67/72] Debugging script --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 07a0b86..51515e3 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -192,7 +192,7 @@ def iterateinitialproductrepos(initialproductcollection, } while( true ) { boolean allparsed = true - products.each { releaserepo, productparsed -> + productcollection.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false parseproductrepo(productcollection, From 54bf29f749811351c1b599422543be8306da0635 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 26 Feb 2022 20:11:59 +0000 Subject: [PATCH 68/72] Attempt to separate dependent products and sub-components --- product/info.groovy | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 51515e3..3d7bc30 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -124,24 +124,29 @@ def parseproductdependencies(LinkedHashMap products, def dependentmeta = productreleasemeta.getassociated(productreleasefolder) for (meta in dependentmeta) { - if (meta.productinfo.subcomponentname != 'WHOLE') { - def pc = new ProductComponent(meta.productinfo.subcomponentname, - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductComponent(pc) - } else { - if (meta.productvariant() != containingproductvariant) { - def pd = new ProductDependency(this, - rt, - meta.productvariant(), - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - pd.setreleasestate() + def pd = new ProductDependency(this, + rt, + meta.productvariant(), + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + pd.setreleasestate() + if (meta.productinfo.subcomponentname == 'WHOLE') { + if (meta.productvariant() != containingproductvariant) { // Don't add myself addProductEntry(products, meta.productvariant(), meta.releaseinfo.releaserepoURL) productrelease.addProductDependency(pd) } + } else if (meta.productvariant() != containingproductvariant) { + addProductEntry(products, + meta.productvariant(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } else { + def pc = new ProductComponent(meta.productinfo.subcomponentname, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductComponent(pc) } } } From 23931461adc32f137b88f013253f645a328c93ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 1 Mar 2022 22:44:22 +0000 Subject: [PATCH 69/72] Use releasetag script --- publish/promptingpublish.groovy | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index d0f6564..053004e 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -52,19 +52,17 @@ pipeline { stage('Identify software release to publish') { steps { script { + rt = load("${autobuilder.commonscripts()}/release/tag.groovy") gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) def releases = new ArrayList() for (release in allreleases) { def boolean canadd = true - if (FILTER_REDUNDANT_RELEASES && ( - release.toLowerCase().startsWith("superceded") || - release.toLowerCase().startsWith("superseded") || - release.toLowerCase().startsWith("erroneous") || - release.toLowerCase().startsWith("faulty") || - release.toLowerCase().startsWith("ignore") || - release.toLowerCase().startsWith("inmainline"))) { - canadd = false + if (FILTER_REDUNDANT_RELEASES) { + def results = rt.startswithspecialprefix(release) + if (results.prefixfound) { + canadd = false + } } if (FILTER_LONGNAME_RELEASES && release.findAll(/[a-zA-Z]/)) { canadd = false From db92ee3e90180bef27289b57e025e79d650ae208 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 1 Mar 2022 23:05:39 +0000 Subject: [PATCH 70/72] Debug release filtering --- publish/promptingpublish.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 053004e..f6002e5 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -62,10 +62,14 @@ pipeline { def results = rt.startswithspecialprefix(release) if (results.prefixfound) { canadd = false + echo "Found prefix in ${release}" } } - if (FILTER_LONGNAME_RELEASES && release.findAll(/[a-zA-Z]/)) { - canadd = false + if (FILTER_LONGNAME_RELEASES) { + if (release.findAll(/[a-zA-Z]/)) { + canadd = false + echo "${release} is a long name" + } } if (canadd) { echo("Showing release: ${release}") From 3730461511700e692f81f2fb780398254307a71c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 1 Mar 2022 23:13:05 +0000 Subject: [PATCH 71/72] Debug release filtering --- publish/promptingpublish.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index f6002e5..7430a3c 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -58,14 +58,14 @@ pipeline { def releases = new ArrayList() for (release in allreleases) { def boolean canadd = true - if (FILTER_REDUNDANT_RELEASES) { + if (FILTER_REDUNDANT_RELEASES == 'true') { def results = rt.startswithspecialprefix(release) if (results.prefixfound) { canadd = false echo "Found prefix in ${release}" } } - if (FILTER_LONGNAME_RELEASES) { + if (FILTER_LONGNAME_RELEASES == 'true') { if (release.findAll(/[a-zA-Z]/)) { canadd = false echo "${release} is a long name" From b70fe5f7bd258ba0c3be2eaeebb12403e01c460b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 1 Mar 2022 23:17:58 +0000 Subject: [PATCH 72/72] Remove debug release filtering --- publish/promptingpublish.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 7430a3c..49d9421 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -62,13 +62,11 @@ pipeline { def results = rt.startswithspecialprefix(release) if (results.prefixfound) { canadd = false - echo "Found prefix in ${release}" } } if (FILTER_LONGNAME_RELEASES == 'true') { if (release.findAll(/[a-zA-Z]/)) { canadd = false - echo "${release} is a long name" } } if (canadd) {