From 25c04061ade18f8eb882a59f9c72aacd0d634a3c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 14:37:54 +0000 Subject: [PATCH 01/73] Initial commit --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..48bbb48 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Local Network Package Updater Autobuild Scripts + +Autobuild scripts for package updater \ No newline at end of file From 1b74ea88ffc6cf2fa93834d9594acceb0787f4e5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 14:42:51 +0000 Subject: [PATCH 02/73] Squashed 'Common_scripts/' content from commit 2c26b22 git-subtree-dir: Common_scripts git-subtree-split: 2c26b221f12ad963c322612f2935afeb2705c68c --- .project | 17 + .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + artifacts/archiver.groovy | 17 + artifacts/metaexport.groovy | 26 + autobuilder.groovy | 147 ++ build/integrity.groovy | 113 ++ build/postactions.groovy | 225 ++++ build/postactionstest.groovy | 43 + check/build_meta.groovy | 87 ++ check/do_meta.groovy | 77 ++ checksum/utils.groovy | 15 + clean/clean.groovy | 10 + commandline/commandline.groovy | 12 + .../eclipsebuild/disableparallelbuilds.groovy | 12 + compile/eclipsebuild/eclipseSTM8build.groovy | 172 +++ compile/eclipsebuild/eclipseds5build.groovy | 252 ++++ compile/eclipsebuild/eclipseniosbuild.groovy | 116 ++ configuration/configuration.groovy | 18 + configuration/tools.groovy | 59 + copy/all.groovy | 79 ++ copy/esw.groovy | 67 + copy/filtered.groovy | 73 + copy/logic.groovy | 79 ++ differences/.project | 17 + differences/.pydevproject | 8 + differences/dometahierarchy.groovy | 836 ++++++++++++ differences/file.groovy | 24 + differences/json.groovy | 51 + differences/metahierarchy.groovy | 102 ++ documentation/doxygen_deploy.groovy | 80 ++ hardware/sominfo.groovy | 80 ++ import/importer.groovy | 377 ++++++ install/installhelper.groovy | 117 ++ meta/folder.groovy | 1193 +++++++++++++++++ packaging/debian.groovy | 111 ++ platform/lineendings.groovy | 5 + publish/.classpath | 9 + publish/.gitignore | 7 + publish/.project | 18 + .../org.eclipse.jdt.groovy.core.prefs | 3 + publish/do.groovy | 174 +++ publish/package.groovy | 181 +++ publish/promptingpublish.groovy | 182 +++ publish/publish.groovy | 92 ++ publish/publishitems.groovy | 129 ++ publishing/utils.groovy | 27 + release/do.groovy | 458 +++++++ release/registerrepo.groovy | 12 + release/release.groovy | 87 ++ remoteaccess/ftp.groovy | 32 + remoteaccess/hostedftp.groovy | 28 + staticanalysis/ccm.groovy | 85 ++ staticanalysis/cpd.groovy | 152 +++ staticanalysis/cppcheck.groovy | 114 ++ test/testethernetproduct.groovy | 204 +++ textutils/subst.groovy | 8 + .../vs2017install/buildmachines.vsconfig | 25 + versioncontrol/git.groovy | 377 ++++++ versionnumbering/do.groovy | 23 + versionnumbering/generate.groovy | 180 +++ versionstamping/.project | 17 + versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 38 + 64 files changed, 7397 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 artifacts/archiver.groovy create mode 100644 artifacts/metaexport.groovy create mode 100644 autobuilder.groovy create mode 100644 build/integrity.groovy create mode 100644 build/postactions.groovy create mode 100644 build/postactionstest.groovy create mode 100644 check/build_meta.groovy create mode 100644 check/do_meta.groovy create mode 100644 checksum/utils.groovy create mode 100644 clean/clean.groovy create mode 100644 commandline/commandline.groovy create mode 100644 compile/eclipsebuild/disableparallelbuilds.groovy create mode 100644 compile/eclipsebuild/eclipseSTM8build.groovy create mode 100644 compile/eclipsebuild/eclipseds5build.groovy create mode 100644 compile/eclipsebuild/eclipseniosbuild.groovy create mode 100644 configuration/configuration.groovy create mode 100644 configuration/tools.groovy create mode 100644 copy/all.groovy create mode 100644 copy/esw.groovy create mode 100644 copy/filtered.groovy create mode 100644 copy/logic.groovy create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/dometahierarchy.groovy create mode 100644 differences/file.groovy create mode 100644 differences/json.groovy create mode 100644 differences/metahierarchy.groovy create mode 100644 documentation/doxygen_deploy.groovy create mode 100644 hardware/sominfo.groovy create mode 100644 import/importer.groovy create mode 100644 install/installhelper.groovy create mode 100644 meta/folder.groovy create mode 100644 packaging/debian.groovy create mode 100644 platform/lineendings.groovy create mode 100644 publish/.classpath create mode 100644 publish/.gitignore create mode 100644 publish/.project create mode 100644 publish/.settings/org.eclipse.jdt.groovy.core.prefs create mode 100644 publish/do.groovy create mode 100644 publish/package.groovy create mode 100644 publish/promptingpublish.groovy create mode 100644 publish/publish.groovy create mode 100644 publish/publishitems.groovy create mode 100644 publishing/utils.groovy create mode 100644 release/do.groovy create mode 100644 release/registerrepo.groovy create mode 100644 release/release.groovy create mode 100644 remoteaccess/ftp.groovy create mode 100644 remoteaccess/hostedftp.groovy create mode 100644 staticanalysis/ccm.groovy create mode 100644 staticanalysis/cpd.groovy create mode 100644 staticanalysis/cppcheck.groovy create mode 100644 test/testethernetproduct.groovy create mode 100644 textutils/subst.groovy create mode 100644 tool configurations/vs2017install/buildmachines.vsconfig create mode 100644 versioncontrol/git.groovy create mode 100644 versionnumbering/do.groovy create mode 100644 versionnumbering/generate.groovy create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy new file mode 100644 index 0000000..bafba02 --- /dev/null +++ b/artifacts/archiver.groovy @@ -0,0 +1,17 @@ +enum Store { + JENKINS, + ARTIFACTSERVER, +} + +this.JENKINS = Store.JENKINS +this.ARTIFACTSERVER = Store.JENKINS + +def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { + echo "Archiving artifacts to store: ${store}" + meta.copyrelatedmeta() + if (store == JENKINS) { + archiveArtifacts(artifacts: artifactlist) + } +} + +return this \ No newline at end of file diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy new file mode 100644 index 0000000..8e5e56e --- /dev/null +++ b/artifacts/metaexport.groovy @@ -0,0 +1,26 @@ +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(meta, importer, subJobs) { + def ArrayList artifactFolders = new ArrayList() + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getProjectName() + def subJobNumber = subJobObject.getNumber() + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] + dir(destFolder) { + deleteDir() + } + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(subJobName, subJobNumber) + importoptions.setincrementversionnumber(incrementversionnumber) + importer.artifactimport(meta, importoptions, destFolder) + + artifactFolders.add("${destFolder}/**") + } + } + return artifactFolders +} + +return this; diff --git a/autobuilder.groovy b/autobuilder.groovy new file mode 100644 index 0000000..667f89a --- /dev/null +++ b/autobuilder.groovy @@ -0,0 +1,147 @@ +// Assume that WORKSPACE environment variable is set + +autobuildroot = 'autobuild' +commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + def stashname = 'Clean workspace stash' + stash(allowEmpty: true, + includes: "${autobuildroot}/**,${autobuildroot}/.git/**", + name: stashname, + useDefaultExcludes: false + ) + cleanWs() + unstash name: stashname + } + else { + echo "Not cleaning workspace" + } +} + +def commonscripts() { + return commonscriptsroot +} + +def getautobuildpath() { + return autobuildroot +} + +def getmetafolderscript() { + return metafolderscript +} + +def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { + def String altered = releaserepo.trim() + + if (releaserepovariantsuffix?.trim()) { + altered = altered.replace("-RELEASE_REPO_VARIANT_SUFFIX-", releaserepovariantsuffix) + } + return altered +} + +def setBuildType(meta, boolean dofullbuild = true) { + def doquickbuild = false + + echo "Build trigger cause: ${currentBuild.getBuildCauses()}" + if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { + echo "Build triggered by user" + if (!dofullbuild) { + echo "Marking as quick build as user wanted one" + doquickbuild = true + } + } else { + echo "Build triggered automatically" + if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { + echo "Marking as quick build as it was triggered by a commit" + doquickbuild = true + } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { + echo "Marking as full build as it was triggered by timer" + } + } + echo "Build type is ${doquickbuild ? "quick" : "full"}" + if (doquickbuild) { + unstable('This not a full build - a subset of the artifacts is not being generated') + } + meta.jenkinsbuildinfo.set(!doquickbuild) + return doquickbuild +} + +def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { + def meta = getmetafolderscript().metareset(script) + echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" + meta.productinfo.set(productname, subcomponentname, productvariant) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + for (p in params) { + meta.jenkinsbuildinfo.addParameter(p.key, p.value) + } + return meta +} + +// Start up boilerplate +def startbuild( + script, + String productname = PRODUCT_NAME, + String subcomponentname = SUBCOMPONENT_NAME, + String releaserepo = RELEASE_REPO, + String productvariant = '', + String autobuildpath = autobuildroot, + String commonscriptspath = commonscriptsroot + ) { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = setupmeta(script, productname, subcomponentname, productvariant) + + def rr = load("${commonscripts()}/release/registerrepo.groovy") + def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) + + echo "Release repo: ${alteredreleaserepo}" + rr.registerrepo(meta, alteredreleaserepo) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +// Start up boilerplate +def startnonbuild( + script, + autobuildpath = autobuildroot, + commonscriptspath = commonscriptsroot + ) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.groovy") +} + +// Everyone wants to know this +autobuilder = this + +return this; diff --git a/build/integrity.groovy b/build/integrity.groovy new file mode 100644 index 0000000..e93c00f --- /dev/null +++ b/build/integrity.groovy @@ -0,0 +1,113 @@ +import static groovy.json.JsonOutput.* + +def check(buildmeta) { + def results = [ + 'good': true, + 'comment': '', + ] + + // Walk the meta looking for different versions of the same components + def allmeta = buildmeta.allrelatedmeta() + echo "All meta:\n${allmeta.comment}" + + // Store build stamps associated with each product component; Store a counter against each product component version + def productcomponents = [:] + for (meta in allmeta.metacollection) { + def key = meta.componentidentifier() + + if (!productcomponents.containsKey(key)) { + productcomponents[key] = [:] + } + def buildstampnumber = meta.releasetag() + if (!productcomponents[key].containsKey(buildstampnumber)) { + productcomponents[key][buildstampnumber] = 1 + } else { + productcomponents[key][buildstampnumber] += 1 + } + } + + // Show what was collected + echo prettyPrint(toJson(productcomponents)) + + // Look for product components with more than one build stamp + for (pc in productcomponents.keySet()) { + def bsns = productcomponents[pc] + if (bsns.size() > 1) { + results.good = false + results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" + for (bsn in bsns.keySet()) { + results.comment += "${bsn}\n" + } + } + } + + // Store product components associated with each release git url + def releasegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.releaseinfo.releaserepoURL + + if (key) { + if (!releasegiturls.containsKey(key)) { + releasegiturls[key] = [:] + } + def componentidentifier = meta.componentidentifier() + if (!releasegiturls[key].containsKey(componentidentifier)) { + releasegiturls[key][componentidentifier] = 1 + } else { + releasegiturls[key][componentidentifier] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(releasegiturls)) + + // Look for release git urls being used by more than one product component + for (rgu in releasegiturls.keySet()) { + def cis = releasegiturls[rgu] + if (cis.size() > 1) { + results.good = false + results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" + for (ci in cis.keySet()) { + results.comment += "${ci}\n" + } + } + } + + // Store git commit shas associated with each source git url + def sourcegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.gitinfo.remoteurl + + if (key) { + if (!sourcegiturls.containsKey(key)) { + sourcegiturls[key] = [:] + } + def commitsha = meta.gitinfo.gitrevision + if (!sourcegiturls[key].containsKey(commitsha)) { + sourcegiturls[key][commitsha] = 1 + } else { + sourcegiturls[key][commitsha] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(sourcegiturls)) + + // Look for source git urls associated with more than one sha code + for (sgu in sourcegiturls.keySet()) { + def cs = sourcegiturls[sgu] + if (cs.size() > 1) { + results.good = false + results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" + for (c in cs.keySet()) { + results.comment += "${c}\n" + } + } + } + + return results +} + +return this \ No newline at end of file diff --git a/build/postactions.groovy b/build/postactions.groovy new file mode 100644 index 0000000..8b18908 --- /dev/null +++ b/build/postactions.groovy @@ -0,0 +1,225 @@ +/* Example post build info +{ + "Build outcomes": { + "Success": { + "Actions": { + "Deploy documentation": { + "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" + }, + "Static analysis": { + "Description": "Can be empty - Static analysis job to run when this build is successful", + "Job": "" + }, + "Unit test": { + "Description": "Can be empty - Unit test job to run when this build is successful", + "Job": "" + }, + "Regression test": { + "Description": "Can be empty - Regression test job to run when this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" + }, + "Update differences": { + "Description": "Can be empty - Update differences job to run when this build is successful", + "Job": "HMB_FD_HV_diff_releases" + }, + "Release to git": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "Job": "HMB_FD_HV_release_to_git" + } + } + }, + "Failed": { + "Actions": { + "Email": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ] + }, + "ExtEmail": { + "Description": "Send an email using Extended Email plugin", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ], + "Recipient providers": [ + "culprits", + "requestor", + "developers", + "upstreamDevelopers" + ] + } + } + } + } +} +*/ + +def getActionJobName(postbuildactions, String action, String alternativeparamname) { + def String jobname = '' + if (postbuildactions) { + try { + jobname = postbuildactions[action]['Job'] + } catch(Exception ex) { + echo "No post build actions found for action: ${action}" + jobname = '' + } + } else { + if (params[alternativeparamname] != null) { + if (params[alternativeparamname]) { + jobname = params[alternativeparamname] + } + } + } + if (jobname) { + def jobitem = Jenkins.instance.getItemByFullName(jobname) + if (jobitem) { + if (!jobitem.isBuildable()) { + jobname = '' + } + } else { + jobname = '' + } + } + // echo "Job name for action: ${action}: ${jobname}" + return jobname +} + +def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { + def jobname = getActionJobName(postbuildactions, action, alternativeparamname) + if (jobname) { + build( + wait: false, + job: jobname, + parameters: [ + string(name: jobnameparametername, value: JOB_NAME), + string(name: jobselectorparametername, value: buildselector), + ] + ) + } +} + +def readPostBuildInfo() { + def postbuildinfo = readJSON(text: POST_BUILD_INFO) + def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) + // echo "Post build info:\n${postbuildjsontext}" + + return postbuildinfo +} + +def performSuccessfulBuildActions() { + def postbuildactions + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Success/Actions" + postbuildactions = null + } + } + def String buildselector = "${BUILD_NUMBER}" + deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') + deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') +} + +def sendemails(emailinfo) { + def emailtolist = emailinfo['To'] + + if (emailtolist) { + def emailbody = """ + Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} + + Console output: ${BUILD_URL}console + """ + for(emailto in emailtolist) { + mail( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: emailbody, + to: emailto + ) + // echo "Send email to: ${emailto}" + } + } +} + +def getRecipientProviders(recipientProviderInfoList) { + def recipientProviderMap = [ + culprits: [$class: 'CulpritsRecipientProvider'], + requestor: [$class: 'RequesterRecipientProvider'], + developers: [$class: 'DevelopersRecipientProvider'], + upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], + ] + def recipientProviders = [] + + if (recipientProviderInfoList) { + for(rp in recipientProviderInfoList) { + if (recipientProviderMap.containsKey(rp)) { + recipientProviders.add(recipientProviderMap[rp]) + } + } + } + if (recipientProviders.size() == 0) { + recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] + } + // echo "recipientProviders: ${recipientProviders.toString()}" + return recipientProviders; +} + +def String getToList(toListInfo) { + def String toList = '' + + if (toListInfo) { + toList = toListInfo.join(',') + } + // echo "toList: ${toList}" + return toList +} + +def sendextemail(extmailinfo) { + def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) + def toList = getToList(extmailinfo['To']) + + emailext( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: '''${JELLY_SCRIPT,template="html"}''', + attachLog: true, + // attachmentsPattern: '', + compressLog: false, + // from : '', + mimeType: 'text/html', + // postsendScript: '', + // presendScript: '', + recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers + // replyTo: '', + saveOutput: false, + to: toList, + ) +} + +def performFailedBuildActions() { + def postbuildactions = null + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Failed/Actions" + postbuildactions = null + } + } + if (postbuildactions) { + if (postbuildactions['Email']) { + sendemails(postbuildactions['Email']) + } + if (postbuildactions['ExtEmail']) { + sendextemail(postbuildactions['ExtEmail']) + } + } +} + +return this; diff --git a/build/postactionstest.groovy b/build/postactionstest.groovy new file mode 100644 index 0000000..150f5d1 --- /dev/null +++ b/build/postactionstest.groovy @@ -0,0 +1,43 @@ +pipeline { + agent { label 'master' } + stages { + stage('Initial setup') { + steps { + script { + CLEAN_WORKSPACE = false + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Dummy build job') { + steps { + script { + echo "Execute dummy build job: ${CAUSE_ERROR}" + if (CAUSE_ERROR == 'true') { + error("Causing error") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() + } + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() + } + } + } +} diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..fb8a75d --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..4b8a89e --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,77 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = entry.metainfo.componentversionuniqueidentifier() + def offsetpath = entry.accumulatedpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + + // Walk the meta information to a max depth + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.accumulatedpath + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..be7569d --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,15 @@ + +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(meta, checksummable) { + for (filepath in checksummable) { + def checksumset = meta.checksuminfo.addChecksumSet(filepath) + def files = findFiles(glob: filepath) + for (file in files) { + echo "Checksumming: ${file.path}" + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) + } + } +} + +return this; \ No newline at end of file diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file diff --git a/commandline/commandline.groovy b/commandline/commandline.groovy new file mode 100644 index 0000000..38f80fa --- /dev/null +++ b/commandline/commandline.groovy @@ -0,0 +1,12 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return powershell(returnStdout: true, script: """ + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.groovy b/compile/eclipsebuild/disableparallelbuilds.groovy new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.groovy @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + + return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/configuration/tools.groovy b/configuration/tools.groovy new file mode 100644 index 0000000..dbaeedb --- /dev/null +++ b/configuration/tools.groovy @@ -0,0 +1,59 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +class ToolOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + String m_toolName + + String m_gitURL + String m_revision + String m_repoRoot + int m_cloneDepth + boolean m_shallowClone + + ToolOptions(script) { + m_script = script + m_gitURL = '-' + m_revision = '-' + m_repoRoot = '' + m_cloneDepth = 1 + m_shallowClone = true + } + + void toolName(String name) { + m_toolName = name.trim() + } + + void toolRepo(String gitURL, String revision) { + m_gitURL = gitURL.trim() + m_revision = revision.trim() + } + + void repoRoot(String root) { + m_repoRoot = root.trim() + } + + void clonedepth(int depth) { + m_cloneDepth = depth + } + + void shallowclone(boolean shallow) { + m_shallowClone = shallow + } +} + +ToolOptions getNewToolOptions() { + return new ToolOptions(this) +} + +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) +} + +def getTool(meta, ToolOptions options, String targetDir) { + gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) + + registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/copy/all.groovy b/copy/all.groovy new file mode 100644 index 0000000..aab479c --- /dev/null +++ b/copy/all.groovy @@ -0,0 +1,79 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + IMPORTS = 'imports' + cleanfolders = [ + IMPORTS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import source artifacts') { + steps { + script { + def copyoptions = importer.getNewImportOptions() + copyoptions.source( + SOURCE_ARTIFACT_LOCATION, + SOURCE_ARTIFACT_BUILD_JOB, + SOURCE_ARTIFACT_BUILD_SELECTOR, + SOURCE_ARTIFACT_RELEASE_REPO, + SOURCE_ARTIFACT_RELEASE_REVISION) + echo "Nested: ${NESTED_COPY}" + copyoptions.setnested(NESTED_COPY == 'true') + importer.artifactimport(meta, copyoptions, IMPORTS) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + + dir(IMPORTS) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + + cleaner.clean([IMPORTS]) + + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/copy/esw.groovy b/copy/esw.groovy new file mode 100644 index 0000000..b9eb737 --- /dev/null +++ b/copy/esw.groovy @@ -0,0 +1,67 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + env.ESW = 'esw' + env.ESWDEST = "${WORKSPACE}/${ESW}" + env.ESWEXPORT = 'embedded_software' + cleanfolders = [ + ESWDEST, + ESWEXPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying embedded software artifacts') { + steps { + script { + def eswimportoptions = importer.getNewImportOptions() + eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) + eswimportoptions.setincrementversionnumber(true) + importer.artifactimport(meta, eswimportoptions, ESWDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) + sh """ + rm -v -rf diff + rm -v -rf meta + mv -v "${ESW}"/* . + """ + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} \ No newline at end of file diff --git a/copy/filtered.groovy b/copy/filtered.groovy new file mode 100644 index 0000000..3ad24cd --- /dev/null +++ b/copy/filtered.groovy @@ -0,0 +1,73 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean') { + steps { + script { + env.IMPORT = "${WORKSPACE}/import" + env.FILTEREDDEST = "${WORKSPACE}/filtered" + cleanfolders = [ + FILTEREDDEST, + IMPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import artifacts') { + steps { + script { + def importoptions = importer.getNewImportOptions() + importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) + importer.artifactimport(meta, importoptions, IMPORT) + } + } + } + stage('Filtered copy') { + steps { + script { + dir(IMPORT) { + fileOperations([ + fileCopyOperation(excludes: EXCLUDES, + flattenFiles: FLATTENOUTPUT, + includes: INCLUDES, + targetLocation: FILTEREDDEST)]) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["filtered/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..db9dbc3 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,79 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + LOGICDEST = 'logic' + cleanfolders = [ + LOGICDEST, + "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/el", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import source artifacts') { + steps { + script { + def logicoptions = importer.getNewImportOptions() + logicoptions.source( + LOGIC_ARTIFACT_LOCATION, + LOGIC_ARTIFACT_PROJECT, + LOGIC_ARTIFACT_BUILD_SELECTOR, + LOGIC_ARTIFACT_RELEASE_REPO, + LOGIC_ARTIFACT_RELEASE_REVISION) + importer.artifactimport(meta, logicoptions, LOGICDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + + dir(LOGICDEST) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + + cleaner.clean([LOGICDEST]) + + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/differences/.project b/differences/.project new file mode 100644 index 0000000..75af0fc --- /dev/null +++ b/differences/.project @@ -0,0 +1,17 @@ + + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy new file mode 100644 index 0000000..3b25798 --- /dev/null +++ b/differences/dometahierarchy.groovy @@ -0,0 +1,836 @@ +import groovy.json.JsonSlurper + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" + +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" + +reportfolder = "${WORKSPACE}/reports" + +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } +} + +class DifferenceRecord { + String productname + String subcomponentname + + String rawdifferencetext + String sha + String type + String mergeinfo + String squashinfo + List jiraticketinfo + String comment + + DifferenceRecord() { + productname = "" + subcomponentname = "" + + rawdifferencetext = "" + sha = "" + type = "" + mergeinfo = "" + squashinfo = "" + jiraticketinfo = [] + comment = "" + } + + def String dump() { + String jiraticketstring = '' + for (j in jiraticketinfo) { + jiraticketstring += j.dump() + " " + } + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + } +} + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } + dir(reportfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def showmetaitems(metaitems) { + String info = '' + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + def accumulatedpath = relatedmetainfo.accumulatedpath + info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" + } + echo info +} + +def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { + boolean checkedout = false + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo?.trim()) { + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } + checkedout = true + } + return checkedout +} + +def String getcomponent(meta) { + def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) + + return [checkedout, uniquefolderforsourcerepo] +} + +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def String releaseexpression + if (tripledot) { + releaseexpression = earlyrevision + "..." + laterrevision + } else { + releaseexpression = earlyrevision + ".." + laterrevision + } + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() + } + return differencetext +} + +def extractsha(String line) { + def pattern = ~"^(\\p{XDigit}*) (.*)" + def match = (line =~ pattern) + match.find() + + def sha = match.group(1) + def restofline = match.group(2) + + //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" + + return [sha.trim(), restofline.trim()] +} + +def extractsquashes(String line) { + def pattern1 = "^(.*)(Squashed .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + return [false, line] +} + +def extractmerges(String line) { + def pattern1 = "^(.*)(Merge branch .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + def pattern2 = "^(.*)(Merge remote-tracking branch .*)" + def match2 = (line =~ pattern2) + if (match2.find()) { + return [true, match2.group(2)] + } + + def pattern3 = "^(.*)(Merge commit .*)" + def match3 = (line =~ pattern3) + if (match3.find()) { + return [true, match3.group(2)] + } + + def pattern4 = "^(.*)(Add .* from commit .*)" + def match4 = (line =~ pattern4) + if (match4.find()) { + return [true, match4.group(2)] + } + + return [false, line] +} + +def jiraticketexists(String jiraticket) { + def String URI = "${JIRA_URI}/browse/${jiraticket}" + def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" + def get = new URL(RestfulURI).openConnection(); + get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) + get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) + get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) + get.setRequestProperty( 'Accept', 'application/json' ) + def getRC = get.getResponseCode(); + + def String redirectedjiraticket = jiraticket + if(!getRC.equals(200)) { + return [false, '', jiraticket] + } else { + def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) + redirectedjiraticket = jsonresponse['key'] + } + return [true, URI, redirectedjiraticket] +} + +def extractjiraticketinfo(String line) { + def List jiratickets = [] + boolean exists = false + String URI = '' + + while (true) { + line = line.trim() + def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line + def commamatch = (line =~ commapattern) + if (commamatch.find()) { + line = commamatch.group(1).trim() + } + def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def match1 = (line =~ pattern1) + if (match1.find()) { + jiraticket = match1.group(1) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match1.group(2) + } else { + break + } + } else { + def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 + def match2 = (line =~ pattern2) + if (match2.find()) { + jiraticket = match2.group(1) + "-" + match2.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match2.group(3) + } else { + break + } + } else { + def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 + def match3 = (line =~ pattern3) + if (match3.find()) { + jiraticket = match3.group(1) + "-" + match3.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match3.group(3) + } else { + break + } + } else { + break + } + } + } + } + boolean result = jiratickets.size() > 0 + return [result, jiratickets, line] +} + +def extractreverts(String line) { + boolean result = true + + def pattern1 = '^(.*)Revert \"(.*)\"' + def match1 = (line =~ pattern1) + if (match1.find()) { + def line2 = match1.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + def pattern2 = '^(.*)Partial Revert \"(.*)\"' + def match2 = (line =~ pattern2) + if (match2.find()) { + def line2 = match2.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + return [false, [], line] +} + +def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() + } + return differencetext +} + +def List fillsquashrecords(String sourcepath, String sha) { + def List differencerecords = [] + + String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) + int count = 0 + squashloglines.split('\n').each { + if (count > 0) { // Ignore first line + def line = it + if (line?.trim()) { + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) + //echo "${dr.dump()}" + } + } + count += 1 + } + return differencerecords +} + +def List fillrecord(meta, line, String sourcepath) { + def List differencerecords = [] + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + + dr.rawdifferencetext = line + + // Example of raw text to process: + // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py + + (sha, line2) = extractsha(line) + dr.sha = sha + + (result, squashinfo) = extractsquashes(line2) + if (result) { + dr.type = "Squash" + dr.squashinfo = squashinfo + squashrecords = fillsquashrecords(sourcepath, sha) + differencerecords.addAll(squashrecords) + } else { + (result, mergeinfo) = extractmerges(line2) + if (result) { + dr.type = "Merge" + dr.mergeinfo = mergeinfo + } else { + (result, jiraticketinfo, comment) = extractreverts(line2) + if (result) { + dr.type = "Revert" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.comment = line2 + } + } + } + } + differencerecords.add(dr) + return differencerecords +} + +def List processtext(meta, String text, String sourcepath) { + def List differencerecords = [] + + text.split('\n').each { + def line = it + if (line?.trim()) { + def List drs = fillrecord(meta, line.trim(), sourcepath) + differencerecords.addAll(drs) + //echo "${dr.dump()}" + } + } + return differencerecords +} + +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + differencerecords = processtext(earlymeta, differencetext, sourcepath) + if (earlyrevision == laterrevision) { + summaryappend(" No change") + } + } else { + summaryappend("The early and later git urls don't match") + echo "The early and later git urls don't match" + result = false; + setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") + } + + return [result, differencerecords] +} + +def deletedcomponent(earlymetaitems, key) { + echo "Component ${key} has been deleted in later version" + summaryappend("Deleted ") +} + +def addedcomponent(latermetaitems, key) { + echo "Component ${key} has been added in later version" + summaryappend("Added ") +} + +class SquashReportRecord { + String comment + int duplicates + + SquashReportRecord() { + comment = '' + duplicates = 0 + } +} + +def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { + dir(reportsubfolder) { + def String squashlist = "${title}\nSquash list:\n" + def squashessorted = squashes.sort() + for (asquash in squashessorted) { + def SquashReportRecord s = asquash.value + squashlist += "${s.comment} (${s.duplicates})\n" + } + writeFile(file: "squashlist.txt", text: squashlist) + } +} + +class MergeReportRecord { + String comment + int duplicates + + MergeReportRecord() { + comment = '' + duplicates = 0 + } +} + +def mergereport(String reportsubfolder, String title, LinkedHashMap merges, boolean iscomponent = true) { + dir(reportsubfolder) { + def String mergelist = "${title}\nMerge list:\n" + def mergessorted = merges.sort() + for (amerge in mergessorted) { + def MergeReportRecord m = amerge.value + mergelist += "${m.comment} (${m.duplicates})\n" + } + writeFile(file: "mergelist.txt", text: mergelist) + } +} + +class CommentsRecord { + String comment + int duplicates + + CommentsRecord() { + comment = '' + duplicates = 0 + } +} + +class RevertsReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + RevertsReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def revertsreport(String reportsubfolder, String title, LinkedHashMap reverts, boolean iscomponent = true) { + dir(reportsubfolder) { + def String revertlist = "${title}\nRevert list:\n" + def revertssorted = reverts.sort() + for (arevert in revertssorted) { + def RevertsReportRecord r = arevert.value + revertlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + revertlist += "${c.comment} (${c.duplicates})\n" + } + revertlist += "\n" + } + writeFile(file: "revertlist.txt", text: revertlist) + } +} + +class TicketedReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + TicketedReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def ticketedreport(String reportsubfolder, String title, LinkedHashMap ticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String ticketedlist = "${title}\nTicketed list:\n" + def ticketedsorted = ticketed.sort() + for (aticketed in ticketedsorted) { + def TicketedReportRecord r = aticketed.value + ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + ticketedlist += "${c.comment} (${c.duplicates})\n" + } + ticketedlist += "\n" + } + writeFile(file: "ticketedlist.txt", text: ticketedlist) + } +} + +class UnticketedReportRecord { + String comment + int duplicates + + UnticketedReportRecord() { + comment = '' + duplicates = 0 + } +} + +def unticketedreport(String reportsubfolder, String title, LinkedHashMap unticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String unticketedlist = "${title}\nUnticketed list:\n" + def unticketedsorted = unticketed.sort() + for (anunticketed in unticketedsorted) { + UnticketedReportRecord u = anunticketed.value + unticketedlist += "${u.comment} (${u.duplicates})\n" + } + writeFile(file: "unticketedlist.txt", text: unticketedlist) + } +} + +class TicketsReportRecord { + String jiraticket + String ticketURI + int duplicates + + TicketsReportRecord() { + jiraticket = '' + ticketURI = '' + duplicates = 0 + } +} + +def ticketsreport(String reportsubfolder, String title, tickets, boolean iscomponent = true) { + dir(reportsubfolder) { + String ticketlist = "${title}\nTicket list:\n" + ticketssorted = tickets.sort() + for (ticketnumber in ticketssorted) { + TicketsReportRecord t = ticketnumber.value + ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" + } + writeFile(file: "ticketlist.txt", text: ticketlist) + } +} + +def differencerecordsreport(String reportsubfolder, String title, differencerecords, boolean iscomponent = true) { + dir(reportsubfolder) { + String recordlist = "${title}\n" + for (dr in differencerecords) { + recordlist += dr.dump() + "\n\n" + } + writeFile(file: "recordlist.txt", text: recordlist) + } +} + +def boolean report(meta, title, differencerecords, boolean iscomponent = true) { + def Map ticketed = [:] + def Map squashes = [:] + def Map merges = [:] + def Map reverts = [:] + def Map unticketed = [:] + def Map tickets = [:] + + for (dr in differencerecords) { + if (dr.type == "Squash") { + def SquashReportRecord s + if (!squashes.containsKey(dr.mergeinfo)) { + s = new SquashReportRecord() + s.comment = dr.squashinfo + s.duplicates = 1 + } else { + s = squashes[dr.squashinfo] + s.duplicates += 1 + } + squashes[dr.squashinfo] = s + } + if (dr.type == "Merge") { + def MergeReportRecord m + if (!merges.containsKey(dr.mergeinfo)) { + m = new MergeReportRecord() + m.comment = dr.mergeinfo + m.duplicates = 1 + } else { + m = merges[dr.mergeinfo] + m.duplicates += 1 + } + merges[dr.mergeinfo] = m + } + if (dr.type == "Revert") { + def RevertsReportRecord r + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!reverts.containsKey(j.jiranumber)) { + r = new RevertsReportRecord() + r.jiraticket = j.jiranumber + r.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + r = reverts[j.jiranumber] + def CommentsRecord c + if (!r.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + c = r.commentscollection[dr.comment] + c.duplicates += 1 + } + r.commentscollection[dr.comment] = c + } + reverts[j.jiranumber] = r + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 + } else { + t = tickets[j.jiranumber] + t.duplicates += 1 + } + tickets[j.jiranumber] = t + } + } + if (dr.type == "Ticketed") { + def TicketedReportRecord t + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!ticketed.containsKey(j.jiranumber)) { + t = new TicketedReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + t = ticketed[j.jiranumber] + def CommentsRecord c + if (!t.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + c = t.commentscollection[dr.comment] + c.duplicates += 1 + } + t.commentscollection[dr.comment] = c + } + ticketed[j.jiranumber] = t + def TicketsReportRecord ti + if (!tickets.containsKey(j.jiranumber)) { + ti = new TicketsReportRecord() + ti.jiraticket = j.jiranumber + ti.ticketURI = j.uri + ti.duplicates = 1 + } else { + ti = tickets[j.jiranumber] + ti.duplicates += 1 + } + tickets[j.jiranumber] = ti + } + } + if (dr.type == "Unticketed") { + UnticketedReportRecord u + if (!unticketed.containsKey(dr.comment)) { + u = new UnticketedReportRecord() + u.comment = dr.comment + u.duplicates = 1 + } else { + u = unticketed[dr.comment] + u.duplicates += 1 + } + unticketed[dr.comment] = u + } + } + String reportsubfolder = "" + if (iscomponent) { + reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" + } else { + reportsubfolder = "${reportfolder}/Combined" + } + mergereport(reportsubfolder, title, merges, iscomponent) + squashreport(reportsubfolder, title, squashes, iscomponent) + revertsreport(reportsubfolder, title, reverts, iscomponent) + ticketedreport(reportsubfolder, title, ticketed, iscomponent) + unticketedreport(reportsubfolder, title, unticketed, iscomponent) + ticketsreport(reportsubfolder, title, tickets, iscomponent) + differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) +} + +def title(earlymeta, latermeta) { + return """ + Differences in: + Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} + between: + then: + Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} + now: + Build date: ${latermeta.jenkinsbuildinfo.builddate} Build stamp: ${latermeta.buildstampinfo.buildstamp} Git revision: ${latermeta.gitinfo.gitrevision} + """ +} + +def compare(earlymetaitems, latermetaitems, key) { + boolean result = true; + echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].metainfo + def latermeta = latermetaitems[key].metainfo + + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + + def boolean repoavailable + def String uniquefolderforsourcerepo + (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) + + List differencerecords = [] + if (repoavailable) { + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + summaryappend(" Items: ${differencerecords.size()} ") + report(earlymeta, title(earlymeta, latermeta), differencerecords) + } else { + summaryappend(" No source repo ") + } + summaryappend("\n") + + return [result, differencerecords]; +} + +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def List differencerecords = [] + + def int md = maxdepth.toInteger() + resetfolders() + preconditions() + + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + + def earlymeta = meta.importrelated(true, earlymetafolder) + def latermeta = meta.importrelated(true, latermetafolder) + + def earlymetaitems = earlymeta.releasewalkerunique(md, '', earlymetafolder) + def latermetaitems = latermeta.releasewalkerunique(md, '', latermetafolder) + + echo "Early items:" + showmetaitems(earlymetaitems) + + echo "Later items:" + showmetaitems(latermetaitems) + + for (key in earlymetaitems.keySet()) { + if (latermetaitems.containsKey(key)) { + (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + differencerecords = differencerecords + componentdifferencerecords + } else { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + deletedcomponent(earlymetaitems, key) + summaryappend("\n") + } + } + for (key in latermetaitems.keySet()) { + if (!earlymetaitems.containsKey(key)) { + summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + addedcomponent(latermetaitems, key) + summaryappend("\n") + } + } + report(meta, title(earlymeta, latermeta), differencerecords, false) + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..ee1b8b3 --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,24 @@ +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { + echo "Comparing file: ${importedfile} with: ${currentfile} option: ${ignorewhitespace}" + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } + if (isUnix()) { + sh """ + diff -c ${wsparameter} "${importedfile}" "${currentfile}" + """ + } + else { + // diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # where diff + # if errorlevel 1 EXIT /b 1 + + diff.exe -c ${wsparameter} "${importedfile}" "${currentfile}" + """ + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy new file mode 100644 index 0000000..dbf47b8 --- /dev/null +++ b/differences/json.groovy @@ -0,0 +1,51 @@ +def comparefiles(importedjsonfile, currentjsonfile) { + echo "Comparing json file: ${importedjsonfile} with: ${currentjsonfile}" + if (isUnix()) { + sh """ + which jq + if [ \$? -ne 0 ]; then exit 1; fi + # jq -S sorts json output + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") + """ + } + else { + // Jq and diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # set + # echo %PATH% + try { + if (Get-Command jq) {} + } + catch { + echo "jq is unavailable" + exit \$false + } + try { + if (Get-Command diff) {} + } + catch { + echo "diff is unavailable" + exit \$false + } + + # echo Sorting json in files to compare + # jq -S sorts json output + jq -S . "${importedjsonfile}" > temp1.txt + if (\$lastexitcode -ge 1) { exit 1 } + jq -S . "${currentjsonfile}" > temp2.txt + if (\$lastexitcode -ge 1) { exit 1 } + + # echo Diffing sorted json files + diff.exe -c temp1.txt temp2.txt + \$lec = \$lastexitcode + + Remove-Item temp1.txt + Remove-Item temp2.txt + + if (\$lec -ge 1) { exit 1 } + """ + } +} + +return this; diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy new file mode 100644 index 0000000..01c1890 --- /dev/null +++ b/differences/metahierarchy.groovy @@ -0,0 +1,102 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + SUMMARYDIR = "${WORKSPACE}/summary" + REPORTDIR = "${WORKSPACE}/reports" + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + "${EARLYDIR}", + "${LATERDIR}", + "${SUMMARYDIR}", + "${REPORTDIR}", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Obtain top level meta') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() + + def earlyimportoptions = importer.getNewImportOptions() + earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) + earlyimportoptions.setincrementversionnumber(false) + earlyimportoptions.setfilter('meta/**') + importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) + + def laterimportoptions = importer.getNewImportOptions() + laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) + laterimportoptions.setincrementversionnumber(false) + laterimportoptions.setfilter('meta/**') + importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) + } + } + } + stage('Diff components') { + steps { + script { + def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,reports/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/documentation/doxygen_deploy.groovy b/documentation/doxygen_deploy.groovy new file mode 100644 index 0000000..5054c02 --- /dev/null +++ b/documentation/doxygen_deploy.groovy @@ -0,0 +1,80 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean imports') { + steps { + script { + env.DOXYGEN = "doxygen" + cleanfolders = [ + DOXYGEN, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Import doxygen artifacts') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + def importoptions = importer.getNewImportOptions() + + importoptions.jenkinsjob(BUILD_JOB_WITH_DOXYGEN_ARTIFACTS, BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR) + importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') + doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) + + productname = doxygenmeta.productinfo.productname + buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix + + echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}" + } + } + } + stage('Copy doxygen to server') { + steps { + script { + doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" + pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}" + echo "Copying from ${doxygensource} to ${pathonserver}" + if (isUnix()) { + sh """#!/bin/bash + rm -rfv "${pathonserver}" + if [ \$? -ne 0 ]; then exit 1; fi + + mkdir -pv "${pathonserver}" + if [ \$? -ne 0 ]; then exit 1; fi + + echo mv -v "${doxygensource}"/ "${pathonserver}" + mv -v "${doxygensource}"/ "${pathonserver}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + error("This job does not run on Windows at the moment") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/hardware/sominfo.groovy b/hardware/sominfo.groovy new file mode 100644 index 0000000..72a3ca9 --- /dev/null +++ b/hardware/sominfo.groovy @@ -0,0 +1,80 @@ +def sominfostore = null + +/* Example expected JSON +{ + "SomType": { + "Som_BA4": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", + "Release repo revision": "" + } + }, + "Som_FC5": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", + "Release repo revision": "" + } + } + } +} +*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: sominfostore, pretty: 1) +} + +def boolean exists() { + return (sominfostore != null) +} + +def set(String jsontext) { + sominfostore = readJSON(text: jsontext) + echo "Som support info:\n${asJsonText()}" +} + +def get() { + return sominfostore +} + +def somtypes() { + return sominfostore['SomType'] +} + +def artifactinfo(somtype) { + return somtype['Artifact info'] +} + +def artifactlocation(artifactinfo) { + return artifactinfo['Location'] +} + +def boolean isjoblocation(artifactinfo) { + return (artifactinfo['Location'] == 'Jenkins job artifact' || + artifactinfo['Location'] == 'Jenkins job artefact') +} + +def boolean isreleaserepolocation(artifactinfo) { + return artifactinfo['Location'] == 'Release repository' +} + +def artifactjob(artifactinfo) { + return [ + "name": artifactinfo['Job'], + "selector": "${artifactinfo['Job selector']}" + ] +} + +def releaserepo(artifactinfo) { + return [ + "repo": artifactinfo['Release repo'], + "revision": artifactinfo['Release repo revision'] + ] +} + +return this \ No newline at end of file diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..bb0d32e --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,377 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + +class ImportOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported + String m_filter // Only import files/paths/folders that match the filter + String m_excludes // Exclude files/paths/folders that match the exclude filter + boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts + boolean m_nested // Import all child artifacts too (identified in the related meta info + boolean m_flatten // Flatten files and folders imported + boolean m_includemetainfilter // Make sure meta is also included in filter + boolean m_clean // Clean import folder before obtaining artifacts + boolean m_appendtoparentmeta // Append to parent meta + String m_sourceselect // Where are the artifacts coming from + + String m_jobname + def m_buildnumberorselector + + String m_repo + String m_revision + + ImportOptions(script, + String sourceselect = 'Jenkins job artifact', + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false, + boolean flatten = false, + boolean includemetainfilter = true, + boolean clean = true, + appendtoparentmeta = true) { + m_script = script + m_sourceselect = sourceselect.trim() + m_incrementversionnumber = incrementversionnumber + m_filter = filter.trim() + m_excludes = excludes.trim() + m_fingerprint = fingerprint + m_nested = nested + m_flatten = flatten + m_includemetainfilter = includemetainfilter + m_clean = clean + m_appendtoparentmeta = appendtoparentmeta + } + + void setsourceselect(String sourceselect) { + sourceselect = sourceselect.trim() + switch (sourceselect) { + case 'Jenkins job artefact': + m_sourceselect = 'Jenkins job artifact' + break + case 'Jenkins job artifact': + case 'Release repository': + m_sourceselect = sourceselect + break + default: + m_script.error("Unknown type of artifact source select: ${sourceselect}") + } + } + + void setincrementversionnumber(boolean incrementversionnumber = true) { + m_incrementversionnumber = incrementversionnumber + } + + void setfilter(String filter) { + m_filter = filter.trim() + } + + void setexcludes(String excludes) { + m_excludes = excludes.trim() + } + + void setfingerprint(boolean fingerprint = true) { + m_fingerprint = fingerprint + } + + void setnested(boolean nested = true) { + m_nested = nested + } + + void flattenimports(boolean flatten = true) { + m_flatten = flatten + } + + void includemetainfilter(boolean includemetainfilter = true) { + m_includemetainfilter = includemetainfilter + } + + void clean(boolean clean = true) { + m_clean = clean + } + + void setappendtoparentmeta(boolean appendtoparentmeta) { + m_appendtoparentmeta = appendtoparentmeta + } + + void jenkinsjob(String jobname, buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: buildnumberorselector + ] + setsourceselect('Jenkins job artifact') + } + + void jenkinsjob(String jobname, int buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: "${buildnumberorselector}" + ] + setsourceselect('Jenkins job artifact') + } + + void releaserepo(String repo, String revision) { + m_repo = repo.trim() + m_revision = revision.trim() + setsourceselect('Release repository') + } + + void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) { + jenkinsjob(jobname, buildnumberorselector) + releaserepo(repo, revision) + setsourceselect(sourceselect) + } +} + +ImportOptions getNewImportOptions() { + return new ImportOptions(this) +} + +def formFilter(ImportOptions importoptions) { + def String filter = importoptions.m_filter + if (filter != '') { + if (importoptions.m_includemetainfilter) { + filter = 'meta/**,' + filter + } + } + echo "Import options filter is: ${filter}" + return filter +} + +def conditionalClean(ImportOptions importoptions, String destpath) { + if (importoptions.m_clean) { + echo "Cleaning ${destpath}" + dir(destpath) { + deleteDir() + } + } +} + +def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Jenkins job artifact') { + error("Import options settings have not been set for: Jenkins job artifact") + return + } + + conditionalClean(importoptions, destpath) + + echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" + echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" + copyArtifacts( + filter: formFilter(importoptions), + excludes: importoptions.m_excludes, + fingerprintArtifacts: importoptions.m_fingerprint, + projectName: importoptions.m_jobname, + selector: importoptions.m_buildnumberorselector, + flatten: importoptions.m_flatten, + target: destpath + ) + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) + if (importoptions.m_incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def rewritemetafileasreleased(meta, String filepath) { + // Modify the meta info to say that is has already been released + meta.markreleased() + meta.saveJsonSerialised(filepath) +} + +def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + conditionalClean(importoptions, destpath) + + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + + echo "Getting associated metadata from path: ${destpath}" + def collectedmetalist = importedmeta.getassociated(destpath) + for(def collectedmeta in collectedmetalist) { + echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}" + } + + echo "Cloning associated repos inside path: ${destpath}" + for(def collectedmeta in collectedmetalist) { + def relatedmeta = collectedmeta.metainfo + def relatedoffsetpath = collectedmeta.accumulatedpath + def cloneurl = relatedmeta.releaseinfo.releaserepoURL + def clonerevision = "refs/tags/${relatedmeta.releasetag()}" + def clonepath = "${destpath}/${relatedoffsetpath}" + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + + gh.stripgithistory(clonepath) + + def metafilepath = "${clonepath}${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(importedmeta, metafilepath) + + if (importoptions.m_incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_sourceselect != 'Release repository') { + error("Import options settings have not been set for: Release repository") + return + } + + conditionalClean(importoptions, destpath) + + def rooturl = importoptions.m_repo + def rootrevision = importoptions.m_revision + echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(importedmeta, metafilepath) + + if (importoptions.m_incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimport(meta, ImportOptions importoptions, String destpath) { + switch (importoptions.m_sourceselect) { + case 'Jenkins job artifact': + case 'Jenkins job artefact': + return artifactimportfrombuildjob(meta, importoptions, destpath) + case 'Release repository': + if (importoptions.m_nested) { + return hierarchyartifactimportfromrepo(meta, importoptions, destpath) + } + return artifactimportfromrepo(meta, importoptions, destpath) + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + +// Deprecated (use above) +def artifactimport(meta, + String sourceselect, + String jenkinsjob, + buildnumberorselector, + String repo, + String revision, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false, + boolean nested = false) { + echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" + def importoptions = importer.getNewImportOptions() + + importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + importoptions.setnested(nested) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + int buildnumber, + String destpath, + boolean incrementversionnumber = true, + String filter = '**', + String excludes = '', + boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" + def importoptions = importer.getNewImportOptions() + + importoptions.jenkinsjob(buildjob, " ${buildnumber}") + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + +// Deprecated (use above) +def artifactimport(meta, + String buildjob, + buildnumberorselector, + String destpath, + boolean incrementversionnumber = true, + String filter = '', + String excludes = '', + boolean fingerprint = false) { + echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" + def importoptions = importer.getNewImportOptions() + + importoptions.jenkinsjob(buildjob, buildnumberorselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(filter) + importoptions.setexcludes(excludes) + importoptions.setfingerprint(fingerprint) + + return artifactimport(meta, importoptions, destpath) +} + +// Import the artifacts of a named child component found below parent meta's hierarchy +def artifactimport(meta, + parentmeta, + String parentmetapath, + String childproduct, + String childsubcomponent, + ImportOptions importoptions, + String destinationfolder) { + // Import from parent meta + def childmeta = meta.getNewMetaObject() + def childmeta2 + echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" + echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" + def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) + if (findresults.matchfound) { + childmeta = findresults.matchedmetaitem.metainfo + if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) { + importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) + } else { + importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) + } + childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) + } else { + error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") + } + return childmeta2 +} + +return this; diff --git a/install/installhelper.groovy b/install/installhelper.groovy new file mode 100644 index 0000000..96a8b2e --- /dev/null +++ b/install/installhelper.groovy @@ -0,0 +1,117 @@ +import hudson.FilePath + +// Transfer file from intermediate position to final position; +// creating necessary folders on the way; +// deleting intermediate file +def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { + def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() + + def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" + def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" + def export_filepath = new hudson.FilePath(channel, abs_export_out) + def export_out_folder = export_filepath.getParent() + def export_out_folder_string = export_out_folder.toString() + def export_out_file = export_filepath.getName() + + echo "Creating folder ${export_out_folder_string}" + export_out_folder.mkdirs() + + echo "Copying from ${pathToIntermediate} to ${abs_export_out}" + def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) + intermediate_filepath.copyTo(export_filepath) + intermediate_filepath.delete() +} + +// Given details about artifacts to import and details of installers to build, import artifacts that are required +def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { + artifactFetched = [:] + + def importoptions = importer.getNewImportOptions() + for (installer in toBuild) { + def artifactKeys = installerDetails.get(installer)[1] + if (artifactKeys instanceof java.lang.String) { + artifactKeys = [ artifactKeys ] + } + for (artifactKey in artifactKeys) { + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + if (artifactFetched.get(artifactKey) == null) { + importoptions.jenkinsjob(sourceProject, sourceselector) + importoptions.setincrementversionnumber(incrementversionnumber) + importoptions.setfilter(toCopy) + importoptions.setexcludes(notToCopy) + + def importmeta = importer.artifactimport(meta, importoptions, destFolder) + artifactFetched.put(artifactKey, importmeta) + } + } + } + return artifactFetched +} + +@NonCPS +def checkISMfilebackend(String xml) { + def String message = "" + + def parsedxml = new XmlParser(true, true, true).parseText(xml) + def pathvariableinfo = [:] + def exitcode = 0 + // Load up a map of path variable and path pairs + // Ignore blank path definitions + // Ignore the workspace definition which should have the only absolute path + parsedxml.each{ table -> + if (table.@name == "ISPathVariable") { + table['row'].each{ row -> + def pathname = row['td'][0].value()[0] + def path = row['td'][1].value()[0] + if (path && pathname != 'WORKSPACE') { + pathvariableinfo[pathname] = path + } + } + } + } + // Check that each path variable is defined beginning with a path variable or the WORKSPACE one + pathvariableinfo.each{ pair -> + message += "pair key: ${pair.key} value: ${pair.value}\n" + def path = pair.value + def pathvarmatcher = path =~ /<(.*)>.*/ + if (pathvarmatcher.matches()) { + def pathvarfound = pathvarmatcher.group(1) + if (pathvarfound != 'WORKSPACE') { + if (!pathvariableinfo.containsKey(pathvarfound)) { + message += "Undefined path variable ${pathvarfound} found\n" + exitcode = 1 + } + } + } else { + message += "Path: ${path} does not begin with a path variable\n" + exitcode = 1 + } + } + return [exitcode, message] +} + +def checkISMfile(String ISMfile) { + echo "Checking ISM file: ${ISMfile}" + def xml = readFile(ISMfile) + (exitcode, message) = checkISMfilebackend(xml) + echo message + if (exitcode > 0) { + error "ISM file errors found" + } else { + echo "Checked ISM file - ok" + } +} + +certificatethumbprints = [:] +certificatethumbprints['floreana'] = '0FDEB0A227982C877146CA595091173AC9655479' // Authoring takes place on this machine - so this is what should be in the ISM file already +certificatethumbprints['hawaii'] = '49FD1B361CFF35014B81940C5B046543045DD34B' + +def selectSigningCertificate(String nodeName, String ISMfile) { + def masterpattern = '0FDEB0A227982C877146CA595091173AC9655479' + echo "Substituting occurrences of ${masterpattern} with ${certificatethumbprints[nodeName]} in ISM file: ${ISMfile}" + + textutils_subst = load("${autobuilder.commonscripts()}/textutils/subst.groovy") + textutils_subst.substitute(ISMfile, masterpattern, certificatethumbprints[nodeName]) +} + +return this; diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..6715145 --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,1193 @@ +import java.nio.file.Paths +import java.nio.file.Path +import groovy.json.JsonSlurperClassic + +abstract class GISJsonSerialised implements Serializable { + def script + + GISJsonSerialised(script) { + this.script = script + } + + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + private String format + private String productname + private String subcomponentname + private String variant + private String uniqueid + + ProductInfo(script, String productname = '', String subcomponentname = '', String variant = '') { + super(script) + this.format = '0.1' + this.productname = productname + this.subcomponentname = subcomponentname + this.variant = variant + this.uniqueid = UUID.randomUUID().toString() + } + + void set(String productname, String subcomponentname, String variant = '') { + // Make sure the fields can be used in git tags + def int pstatus = 0 + if (script.isUnix()) { + pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) + } + if (pstatus > 0) { + script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") + } + def int sstatus = 0 + if (script.isUnix()) { + sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) + } + if (sstatus > 0) { + script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") + } + if (variant) { + def int vstatus = 0 + if (script.isUnix()) { + vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) + } + if (vstatus > 0) { + script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") + } + } + this.productname = productname + this.subcomponentname = subcomponentname + this.variant = variant + } + + void set(ProductInfo otherinfo) { + this.productname = otherinfo.productname + this.subcomponentname = otherinfo.subcomponentname + this.variant = otherinfo.variant + this.uniqueid = otherinfo.uniqueid + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + variant = rawmeta['variant'] ?: '' + uniqueid = rawmeta['uniqueid'] ?: uniqueid + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'variant' : variant, + 'uniqueid': uniqueid, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String format + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = '', sourcerevision = '', gitrevision = '', remoteurl = '', remoteinfo = '') { + super(script) + this.format = '0.1' + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + GitInfo(GitInfo another) { + super(another.script) + this.sourcepath = another.sourcepath + this.sourcerevision = another.sourcerevision + this.gitrevision = another.gitrevision + this.remoteurl = another.remoteurl + this.remoteinfo = another.remoteinfo + } + + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + void set(GitInfo otherinfo) { + this.sourcepath = otherinfo.sourcepath + this.sourcerevision = otherinfo.sourcerevision + this.gitrevision = otherinfo.gitrevision + this.remoteurl = otherinfo.remoteurl + this.remoteinfo = otherinfo.remoteinfo + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildParameter extends GISJsonSerialised { + private String name + private String valuetype + private String value + + JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { + super(script) + this.name = name + this.valuetype = valuetype + this.value = value + } + + void set(String name, String valuetype, String value) { + this.name = name + this.valuetype = valuetype + this.value = value + } + + void deserialiseJson(rawmeta) { + name = rawmeta['name'] + valuetype = rawmeta['valuetype'] + value = rawmeta['value'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'name' : name, + 'valuetype' : valuetype, + 'value' : value, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String format + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + private ArrayList parameters + private boolean isfullbuild + + JenkinsBuildInfo(script, String buildjobname = '', int buildnumber = 0, String buildmachine = '', String builddate = '', boolean isfullbuild = true) { + super(script) + this.format = '0.1' + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + this.isfullbuild = isfullbuild + parameters = new ArrayList() + } + + void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + this.isfullbuild = isfullbuild + } + + void set(boolean isfullbuild) { + this.isfullbuild = isfullbuild + } + + boolean isFullBuild() { + return isfullbuild + } + + void addParameter(String name, value) { + def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) + parameters.add(p) + return p + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] + parameters = new ArrayList() + if (rawmeta['isfullbuild'] != null) { + for (rawparameter in rawmeta['parameters']) { + def p = new JenkinsBuildParameter(script) + p.deserialiseJson(rawparameter) + parameters.add(p) + } + } + } + + List parametersserialised() { + def associated = [] + for (p in parameters) { + associated.add(p.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + 'isfullbuild' : isfullbuild, + 'parameters' : parametersserialised(), + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + private String format + private String buildstampprefix + private int revision + private String buildstamp + + BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { + super(script) + this.format = '0.1' + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + } + + BuildStampInfo(BuildStampInfo another) { + super(another.script) + this.buildstampprefix = another.buildstampprefix + this.revision = another.revision + this.buildstamp = another.buildstamp + } + + void set(String buildstampprefix, int revision, String buildstamp) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + } + + void set(BuildStampInfo other) { + set(other.buildstampprefix, other.revision, other.buildstamp) + } + + // Override the buildstamp + void override(String buildstamp) { + this.buildstamp = buildstamp + script.echo("Build stamp overridden to: ${this.buildstamp}") + def s = this.buildstamp.split('\\.') + if (s.size() != 4) { + script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") + } + this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" + script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") + this.revision = s[3].toInteger() + script.echo("Revision overridden to: ${this.revision}") + } + + String buildstampcommasep() { + def String buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + + void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") + revision = newrevision + def List buildstampsplit = buildstamp.split('\\.') + buildstampsplit[-1] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + script.echo("Build stamp now ${buildstamp}") + } + + void set(String buildstampprefix, int revision) { + buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' + this.buildstampprefix = buildstampprefix + this.buildstamp = buildstampprefix + "0" + changerevision(revision) + } + + void set(String buildstampprefix, String revision = '0') { + set(buildstampprefix, revision.toInteger()) + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + private String format + private String releaserepoURL + private ArrayList releaseitems + private boolean released + + ReleaseInfo(script, String releaserepoURL = '', boolean released = false) { + super(script) + this.format = '0.1' + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() + this.released = released + } + + void set(String releaserepoURL = '') { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + script.echo("Adding release item: ${releaseitem} to release items") + releaseitems.add(releaseitem) + } + + void markreleased() { + this.released = true + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + released = rawmeta['released'] ?: false + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), + 'released' : released + ] + return data + } +} + +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = '', configurationURL = '', configurationversion = '', configurationSHA = '') { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath = '', String sum = '') { + super(script) + this.filepath = filepath + this.sum = sum + } + + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String format + private String artifactpath + private ArrayList checksumpairs + + ChecksumSet(script, artifactpath = '') { + super(script) + this.format = '0.1' + this.artifactpath = artifactpath + checksumpairs = new ArrayList() + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } +} + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + checksumsets = new ArrayList() + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + +class PublishableInfo extends GISJsonSerialised { + private String format + private String filepath + + PublishableInfo(script, String filepath = '') { + super(script) + this.format = '0.1' + this.filepath = filepath + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + filepath = rawmeta['filepath'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'filepath' : filepath, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private static String METAFOLDERNAME = 'meta' + private static String METAFILENAME = 'info.json' + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String format = '0.2' + + abstract class RelatedMetaInfo extends GISJsonSerialised { + String format + + RelatedMetaInfo(script, format) { + super(script) + this.format = format + } + + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + abstract String getrelatedid() + abstract void markreleased() + } + + class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { + private MetaInfo meta + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_1(script, String offsetpath = '', MetaInfo meta = null, String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.1') + this.offsetpath = offsetpath + this.uniqueid = uniqueid + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] ?: uniqueid + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid, + 'meta' : meta.serialisationdata(), + ] + return data + } + + String getrelatedid() { + return uniqueid + } + + void markreleased() { + meta.releaseinfo.markreleased() + } + } + + class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { + private String importpath + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_2(script, String importpath = '', String offsetpath = '', String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.2') + this.importpath = importpath + this.offsetpath = offsetpath + this.uniqueid = uniqueid + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid + ] + return data + } + + String getrelatedid() { + return uniqueid + } + + void markreleased() { + // Do nothing + } + } + + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + private ChecksumInfo checksuminfo + private ArrayList configurationinfo + private ArrayList publishableinfo + private LinkedHashMap custominfo + private ArrayList associatedmeta + + MetaInfo(script) { + super(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) + configurationinfo = new ArrayList() + publishableinfo = new ArrayList() + custominfo = new LinkedHashMap() + associatedmeta = new ArrayList() + } + + void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") + releaseinfo.markreleased() + for (associated in associatedmeta) { + associated.markreleased() + } + } + + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return configuration + } + + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") + publishableinfo.add(pi) + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") + configurationinfo.add(ci) + return ci + } + + void addCustomInfo(String customId, String customValue) { + custominfo.put(customId.trim(), customValue) + } + + String getCustomInfo(String customId) { + return custominfo.getAt(customId.trim()) + } + + def isFullBuild() { + return jenkinsbuildinfo.isFullBuild() + } + + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(metaitem.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { + script.error("Unsupported meta format found") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + custominfo = rawmeta['custominfo'] + for (rawrelatedmeta in rawmeta['associatedmeta']) { + if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { + def rm = new RelatedMetaInfo_v0_1(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } else if (rawrelatedmeta['format'] == '0.2') { + def rm = new RelatedMetaInfo_v0_2(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + } + + def deserialiseJson(String filepath) { + script.echo("Deserialising JSON at file: ${filepath}") + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + def deserialiseJsonSlurper(String filepath) { + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) + deserialiseJson(rawmeta) + } + + String relativize(String mypath) { + String pathRelative + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + } else { + pathRelative = mypath + script.echo("Path relative is unchanged: ${pathRelative}") + } + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') + // Strip leading / if any + pathRelative = pathRelative.replaceAll('^/+', '') + + script.echo("Path relative is ${pathRelative}") + + return pathRelative + } + + static String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + + MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { + MetaInfo mi = new MetaInfo(script) + + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + metafilepath() + if (script.fileExists(importedmetafile)) { + script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") + mi.deserialiseJson(importedmetafile) + + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") + addConfigurationInfo('', givenRepo, givenRevision, givenSHA) + } + return mi + } + + def copyrelatedmeta() { + script.echo("Copying related input meta data to output meta") + for (relatedmetainfo in associatedmeta) { + def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" + script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") + script.fileOperations([ + script.folderCreateOperation(folderPath: destpath), + script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) + ]) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), + 'custominfo' : custominfo, + 'associatedmeta' : associatedmetaserialised(), + ] + return data + } + + static String folder() { + return METAFOLDERNAME + } + + def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = METAARTIFACTS + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } + saveJsonSerialised(metafilepath) + return artifactlist + } + + String componentidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + } + + String componentversionuniqueidentifier() { + return componentidentifier() + "_" + buildstampinfo.buildstamp + } + + String globalreleasetag() { + return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber + } + + String releasetag() { + return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber + } + + String releasetagfordebpackage() { + return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber + } + + class CollectedMeta { + private MetaInfo metainfo + private String accumulatedpath + private String accumulatedmetapath + + CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { + this.metainfo = metainfo + this.accumulatedpath = accumulatedpath + this.accumulatedmetapath = accumulatedmetapath + } + } + + class IterationInfo { + private ArrayList metacollection + private Map metamap + private String comment + private int level + private String gatheredpath + private String gatheredmetapath + private boolean limiteddepth + private int maxdepth + + IterationInfo(ArrayList metacollection, + Map metamap, + String comment = '', + int level = 0, + String gatheredpath = '', + String gatheredmetapath = '', + boolean limiteddepth = false, + int maxdepth = 0) { + this.metacollection = metacollection + this.metamap = metamap + this.comment = comment + this.level = level + this.gatheredpath = gatheredpath + this.gatheredmetapath = gatheredmetapath + this.limiteddepth = limiteddepth + this.maxdepth = maxdepth + } + + String show() { + return "Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}" + } + + void add(CollectedMeta item) { + metacollection.add(item) + metamap.put(item.metainfo.productinfo.uniqueid, item) + } + } + + def openchildmeta(IterationInfo ii, RelatedMetaInfo relatedmetainfo) { + def MetaInfo mi = new MetaInfo(script) + + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" + def boolean imported + // If this has already been read in from a file + if (relatedmetainfo.getrelatedid() in ii.metamap) { + //script.echo("Already read in meta associated with uniqueid: ${relatedmetainfo.getrelatedid()} - Using it") + mi = ii.metamap[relatedmetainfo.getrelatedid()].metainfo + imported = true + } else if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + imported = true + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") + imported = false + } + + def childmetainfo = [ + 'imported': imported, + 'nextgatheredpath': nextgatheredpath, + 'nextgatheredmetapath': nextgatheredmetapath, + 'childmeta': mi + ] + return childmetainfo + } + + def walkrelated_v0_1(IterationInfo ii) { + //script.echo("Walk related v0.1: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } else { + script.error("There is non v0.1 format related meta info under v0.1 related meta info") + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + def walkrelated(IterationInfo ii) { + //script.echo("Walk related: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + //script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") + if (relatedmetainfo.format == '0.2') { + def childmetainfo = openchildmeta(ii, relatedmetainfo) + if (childmetainfo.imported) { + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + childmetainfo.nextgatheredpath, + childmetainfo.nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = childmetainfo.childmeta.walkrelated(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + // Walks all layers of associated data only + def allrelatedmeta(String metapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + return walkrelated(new IterationInfo(metacollection, metamap, '', 0, '', metapath)) + } + + ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) + script.echo("Release walker iteration info at start:\n${ii.show()}") + def results = walkrelated(ii) + return results.metacollection + } + + // Walks 1 layer of associated data only + def getassociated(String rootpath) { + return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself) + } + + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + def existingentrypathlength = existingentry.accumulatedpath.split('/').size() + def newentrypathlength = metaitem.accumulatedpath.split('/').size() + if (existingentrypathlength > newentrypathlength) { + releasemetacollection.put(uniquestring, metaitem) + } + } else { + releasemetacollection.put(uniquestring, metaitem) + } + return releasemetacollection + } + + Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") + + def Map versionuniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() + versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) + } + + return versionuniquecollection + } + + Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") + + def Map uniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentidentifier() + uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) + } + + return uniquecollection + } + + MetaInfo getNewMetaObject(script) { + return new MetaInfo(script) + } + + def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { + def results = [ + 'matchedmetaitem' : new CollectedMeta(this), + 'matchfound' : false + ] + def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") + for (dependentmetaitem in walkresults.metacollection) { + script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") + if (dependentmetaitem.metainfo.productinfo.productname == productname && + dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { + script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") + results.matchedmetaitem = dependentmetaitem + results.matchfound = true + break + } + } + return results + } +} + +MetaInfo metareset(script) { + def mi = new MetaInfo(script) + + echo "Resetting meta folder" + dir(mi.folder()) { + deleteDir() + } + + return mi +} + +MetaInfo metafromfolder(String folder = '') { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + +MetaInfo metafromfolderSlurper(String folder = '') { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + +return this; diff --git a/packaging/debian.groovy b/packaging/debian.groovy new file mode 100644 index 0000000..9f2f4d3 --- /dev/null +++ b/packaging/debian.groovy @@ -0,0 +1,111 @@ +class DebRootDefinition { + String ROOTPATH + String PACKAGE_PREFIX + String PACKAGE_FOLDER_NAME + String PACKAGE_NAME + String UNIQUE_PACKAGE_NAME + + String PACKAGE_SOURCE_ROOT + String PACKAGE_SOURCE_FOLDER_NAME + String DEBPATH + + String ETC + String GIS_OPT_BIN + String GIS_OPT_LIB + String GIS_OPT_SCRIPTS_APP + String GIS_OPT_SCRIPTS_EL + String GIS_OPT_SCRIPTS_CONTROL + String GIS_OPT_WWW + String GIS_OPT_OTHER + String GIS_OPT_KERNEL + String GIS_OPT_LOGIC + String GIS_OPT_PRINTHEAD + String GIS_OPT_BOOT + + String USR_BIN + String USR_LIB_ARCH + String LIB_FIRMWARE + String HTTP_DEV_DIV + String LIB_SYSTEMD + + DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix) { + ROOTPATH = rootpath + PACKAGE_PREFIX = packageprefix + PACKAGE_FOLDER_NAME = packageprefix + PACKAGE_NAME = packageprefix + "_armhf.deb" + UNIQUE_PACKAGE_NAME = packageprefix + "_" + buildstamp + "_armhf.deb" + + PACKAGE_SOURCE_ROOT = packagesourceroot + PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" + DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" + + ETC = "${DEBPATH}/etc" + inpackagepathsuffix + GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + inpackagepathsuffix + GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + inpackagepathsuffix + GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + inpackagepathsuffix + GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + inpackagepathsuffix + GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" + inpackagepathsuffix + GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + inpackagepathsuffix + GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + inpackagepathsuffix + GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" + inpackagepathsuffix + GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + inpackagepathsuffix + GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + inpackagepathsuffix + GIS_OPT_BOOT = "${DEBPATH}/opt/gis/boot" + inpackagepathsuffix + + USR_BIN = "${DEBPATH}/usr/bin" + inpackagepathsuffix + USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + inpackagepathsuffix + LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + inpackagepathsuffix + HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + inpackagepathsuffix + LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + inpackagepathsuffix + } +} + +def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix = '') { + return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) +} + +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { + withCredentials([usernamePassword(credentialsId: credentials_id, + usernameVariable: 'SUDO_USERNAME', + passwordVariable: 'SUDO_PASSWORD')]) { + def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" + def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" + try { + sh """#!/bin/bash + # Exit if any command fails + set -xe + + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + + ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" + + mkdir -v -p "${packageout}" + echo "Building package:" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + + mkdir -v -p "${uniquepackageout}" + cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" + + echo "Showing package:" + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package contents:" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + + echo "Showing package lintian output:" + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true + + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + """ + } + catch (Exception e) { + echo "Exception occurred" + sh """#!/bin/bash -v + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + + exit 1 + """ + } + } +} + +return this; \ No newline at end of file diff --git a/platform/lineendings.groovy b/platform/lineendings.groovy new file mode 100644 index 0000000..9657b0d --- /dev/null +++ b/platform/lineendings.groovy @@ -0,0 +1,5 @@ + +// Put ${nl} into strings when wanting a newline +nl = System.properties['line.separator'] + +return this; diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..ea090f0 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,174 @@ +import groovy.io.FileType + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(filteredreleaserepofolder) { + deleteDir() + } +} + +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { + return "Software/${productline} ${softwarerelease}/${productuniquefolder(releasemeta)}" +} + +def uniqueabsoluterreleasefolder(String productline, String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(productline, softwarerelease, releasemeta)}" +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def guessreporootsuffix(componentURL) { + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + return guess +} + +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.metainfo + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" + def publishableitems = releasemeta.publishableinfo + + def ArrayList publishablepaths = new ArrayList() + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + } + } + if (publishablepaths.size() > 0) { + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone + } + else { + summaryappend("None ") + } +} + +def publish(hostedftpsiteinfo, String customer, String finalreleaserepofolder = releaserepofolder) { + summaryappend(" FTP operations: ") + + def String ftproot = "${hostedftpsiteinfo.workingdirectory}${customer}" + dir(finalreleaserepofolder) { + summaryappend("FTP operations: Transferring to ${ftproot}") + sh """#!/bin/bash + ncftpput -u ${hostedftpsiteinfo.username} -p ${hostedftpsiteinfo.password} -R ${hostedftpsiteinfo.url} "${ftproot}" . + """ + summaryappend(", Transferred ") + } +} + +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdirs(); + } + dirFrom.eachFile(FileType.FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { + def releasemeta = releaseinfo.metainfo + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) + + summaryappend("\n") +} + +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + def productline = releasemeta.productinfo.productname + // Walk the meta information to a max depth, looking for artifacts to publish + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) + + echo "Version unique components:" + def String versionuniquecomponent = "" + for (key in componentReleaseInfoVersionUnique.keySet()) { + versionuniquecomponent += "${key}\n" + } + echo versionuniquecomponent + + def itempathstopublish = convertitemstopublish(itemstopublishstring) + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + recallcomponent(productline, releaseinfo, softwarerelease, itempathstopublish) + } + gh.stripgithistory(releaserepofolder) + publish(hostedftpsiteinfo, customer) + + return [summarystring, markunstable, unstablereason] +} + +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(hostedftpsiteinfo, releasemeta, releasemetapath, '', customer, softwarerelease, maxdepth) +} + +return this; diff --git a/publish/package.groovy b/publish/package.groovy new file mode 100644 index 0000000..68c41f3 --- /dev/null +++ b/publish/package.groovy @@ -0,0 +1,181 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHONBRANCH = "origin/3.6.3_linux" + } + else { + PYTHONBRANCH = "origin/3.6.3_windows" + } + PYTHONDIR = "${WORKSPACE}/python" + PYTHONROOTDIR = 'python' + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + } + stage('Clean exports') { + steps { + script { + PACKAGEDEST = "${WORKSPACE}/package" + cleanfolders = [ + PACKAGEDEST, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying package artifacts') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + def packageimportoptions = importer.getNewImportOptions() + packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) + packageimportoptions.setincrementversionnumber(true) + packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) + + PACKAGE_FOLDER = "${PACKAGEDEST}/package" + UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" + } + } + } + stage('Aptly environment') { + steps { + script { + echo "Aptly config file is ${APTLY_CONFIG_FILE}" + sh """#!/bin/bash -v + aptly config show + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Create repo - if not already there') { + steps { + script { + echo "Repo name is ${REPO_NAME}" + sh """#!/bin/bash -v + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then + echo "aptly repo ${REPO_NAME} does not exist; Creating it" + aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + fi + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Drop published repo - if any') { + steps { + script { + sh """#!/bin/bash -v + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + + stage('Add/update package in repo') { + steps { + script { + echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" + sh """#!/bin/bash -v + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Create snapshot') { + steps { + script { + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" + // Convert prefix to regular expression + PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") + PACKAGE_VERSION_REGEX = '' + i = 0 + while(true) { + if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + break; + } + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + PACKAGE_VERSION_REGEX += '\\.' + i++ + } + sh """#!/bin/bash -v + # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' + if [ \$? -ne 0 ]; then exit 1; fi + aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Publish snapshot') { + steps { + script { + sh """#!/bin/bash -v + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy new file mode 100644 index 0000000..ce6a805 --- /dev/null +++ b/publish/promptingpublish.groovy @@ -0,0 +1,182 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "origin/master" + SOURCEDIR = "${WORKSPACE}/whole" + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + } + } + } + stage('Identify customer to publish to') { + steps { + script { + hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") + def List customers = hostedftb.getCustomerList() + timeout(time: 10, unit: 'MINUTES') { + customer = input( + message: 'Please select customer to publish to', + ok: 'Ok', + parameters: [ + choice(name: 'customer', choices: customers, description: 'Which customer?') + ] + ) + echo "Chosen customer is: ${customer}" + } + } + } + } + stage('Identify software release to publish') { + steps { + script { + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + def allreleases = gh.gettaglist(SOURCEDIR) + def releases = new ArrayList() + if (FILTER_REDUNDANT_RELEASES == 'false') { + echo("Showing all releases") + releases = allreleases + } else { + echo("Showing filtered releases") + for (release in allreleases) { + if (!release.toLowerCase().startsWith("superceded") && + !release.toLowerCase().startsWith("superseded") && + !release.toLowerCase().startsWith("erroneous") && + !release.toLowerCase().startsWith("faulty") && + !release.toLowerCase().startsWith("ignore") && + !release.toLowerCase().startsWith("inmainline")) { + echo("Showing release: ${release}") + releases.add(release) + } else { + echo("Not showing release: ${release}") + } + } + } + timeout(time: 10, unit: 'MINUTES') { + release = input( + message: 'Please select the software version', + ok: 'Ok', + parameters: [ + choice(name: 'release', choices: releases, description: 'Which software release?') + ] + ) + echo "Chosen software version is: ${release}" + } + } + } + } + stage('Checkout release repo at selected software version') { + steps { + script { + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${release}" + SOURCEDIR = "${WORKSPACE}/whole" + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + } + } + } + stage('Get release artifact info') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Select which items from software release to publish') { + steps { + script { + publishitems = load("${autobuilder.commonscripts()}/publish/publishitems.groovy") + timeout(time: 10, unit: 'MINUTES') { + def itemsobject = publishitems.createItemSelector( + 'Select software items', + 'Software items', + releasemeta, + SOURCEDIR, + ITEMSDEFAULTLIST, + MAX_META_DEPTH, + '20', + 'PT_MULTI_SELECT') + selecteditems = input( + message: 'Select which items from the release are to be published', + ok: 'Ok', + parameters: [ + itemsobject.multiSelect() + ] + ) + echo "Chosen items are: ${selecteditems}" + } + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, SOURCEDIR, selecteditems, customer, release, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..ad33df9 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,92 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + def hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") + def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/publishitems.groovy b/publish/publishitems.groovy new file mode 100644 index 0000000..9f6c368 --- /dev/null +++ b/publish/publishitems.groovy @@ -0,0 +1,129 @@ +import com.cwctravel.hudson.plugins.extended_choice_parameter.ExtendedChoiceParameterDefinition + +class PublishableItemsSelection { + def script + def ArrayList preselectedmenuitems + def ArrayList menuitemdescriptions + def ArrayList menuitemsfull + def ExtendedChoiceParameterDefinition multiselect + + PublishableItemsSelection(script) { + this.script = script + preselectedmenuitems = new ArrayList() + menuitemdescriptions = new ArrayList() + menuitemsfull = new ArrayList() + } + + def init( + String name, + String description, + releasemeta, + String releasemetapath, + String preselectedext, + String metalevelsstringext, + String visibleItemCountExt, + String controltype) { + outputSelectionfromPreselect(preselectedmenuitems, preselectedext) + def Integer metalevelsext = metalevelsstringext.toInteger() + def Integer visibleItemCount = visibleItemCountExt.toInteger() + def metaitems = releasemeta.releasewalkerunique(metalevelsext, '', releasemetapath) + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + outputDescription(menuitemdescriptions, releaseinfo) + outputSelection(menuitemsfull, releaseinfo) + } + + script.echo("name is: ${name}") + script.echo("description is: ${description}") + script.echo("menuitemsfull is: ${menuitemsfull}") + script.echo("preselectedmenuitems is: ${preselectedmenuitems}") + script.echo("menuitemdescriptions is: ${menuitemdescriptions}") + script.echo("visibleItemCount is: ${visibleItemCount}") + script.echo("controltype is: ${controltype}") + + def seperator = "," + visibleItemCount = Math.min(visibleItemCount, menuitemsfull.size()) + script.echo("visibleItemCount is now: ${visibleItemCount}") + + multiselect = new ExtendedChoiceParameterDefinition( + name, // String name + controltype, // String type + menuitemsfull.join(seperator), // String value + "project name", // String projectName + "", // String propertyFile + "", // String groovyScript + "", // String groovyScriptFile + "", // String bindings + "", // String groovyClasspath + "", // String propertyKey + preselectedmenuitems.join(seperator),// String defaultValue + "", // String defaultPropertyFile + "", // String defaultGroovyScript + "", // String defaultGroovyScriptFile + "", // String defaultBindings + "", // String defaultGroovyClasspath + "", // String defaultPropertyKey + menuitemdescriptions.join(seperator),// String descriptionPropertyValue + "", // String descriptionPropertyFile + "", // String descriptionGroovyScript + "", // String descriptionGroovyScriptFile + "", // String descriptionBindings + "", // String descriptionGroovyClasspath + "", // String descriptionPropertyKey + "", // String javascriptFile + "", // String javascript + false, // boolean saveJSONParameterToFile + false, // boolean quoteValue + visibleItemCount, // int visibleItemCount + description, // String description + seperator) // String multiSelectDelimiter + } + + def outputSelectionfromPreselect(menuitems, preselectedext) { + script.echo ("preselectext: ${preselectedext}") + def menuitemstemp = preselectedext.split(',') + script.echo ("Menuitemstemp: ${menuitemstemp}") + + for (menuitem in menuitemstemp) { + menuitems.add(menuitem.trim()) + } + } + + def outputDescription(menuitems, releaseinfo) { + def meta = releaseinfo.metainfo + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } + } + + def outputSelection(menuitems, releaseinfo) { + def meta = releaseinfo.metainfo + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } + } + + def multiSelect() { + return multiselect + } +} + +PublishableItemsSelection createItemSelector( + String name, + String description, + releasemeta, + String releasemetapath, + String preselectedext, + String metalevelsstringext, + String visibleItemCountExt = '20', + String type = 'PT_CHECKBOX') { // Could also be "PT_MULTI_SELECT" + PublishableItemsSelection pis = new PublishableItemsSelection(this) + + pis.init(name, description, releasemeta, releasemetapath, preselectedext, metalevelsstringext, visibleItemCountExt, type) + + return pis +} + +return this; diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..2daf6f6 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,27 @@ + + +// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout +def allowed(String filepath) { + if (filepath.endsWith('/**') || filepath.endsWith('/')) { + def files = findFiles glob: filepath + if (files.length > 0) { + return + } else { + error "There are no files/folders under the publishable item: ${filepath}" + } + } + error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" +} + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(meta, publishable) { + filelist = [] + for (filepath in publishable) { + if (filepath != meta.METAARTIFACTS) { + allowed(filepath) + } + meta.addPublishable(filepath) + } +} + +return this; diff --git a/release/do.groovy b/release/do.groovy new file mode 100644 index 0000000..4dcb091 --- /dev/null +++ b/release/do.groovy @@ -0,0 +1,458 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +repoandtagsfile = "${summaryfolder}/repoandtags.txt" +repoandtagsstring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") + repoandtagsstring = "" + writeFile(file: repoandtagsfile, text: "Repository summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def repoandtagsappend(String summary) { + def String repoandtagscontents = readFile(file: repoandtagsfile) + writeFile(file: repoandtagsfile, text: repoandtagscontents + summary) + repoandtagsstring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + repoandtagsappend(" Release repo: ${releaserepo}") + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files - Note that this does not remove .git* + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo) { + repoandtagsappend(" Source repo: ${sourcerepo}") + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } else { + repoandtagsappend(" Source repo: None") + } +} + +def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + def boolean result = true + for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { + def filterpath = artifactfilter + if (buildinfo.accumulatedpath) { + filterpath = buildinfo.accumulatedpath + '/' + artifactfilter + } + // Copy relevant artifacts to temp folder + echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + try { + copyArtifacts( + filter: filterpath, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + ], + target: tempfolder + ) + } catch (Exception e) { + echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" + result = false + summaryappend("Does not exist:") + } + if (result) { + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath + dir(originalrootdir) { + sh("mv -v * ${repofolder}") + } + } + summaryappend("${artifactfilter},") + } + return result +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + + return changed +} + +def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem) + } + } + return taglist +} + +def String releasegitgetshacode(String tag) { + def existingshacode = "" + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + existingshacode = readFile(file: gitrevparseoutputfile).trim() + } + return existingshacode +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def String existingshacode = releasegitgetshacode(tag) + if (existingshacode) { // Tag exists + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with the one expected + // This is a problem - The tag is already at another position in git repo + setunstable("The release tag ${tag} has been used at another commit other than one expected - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${postcommitshacode})") + summaryappend("(${tag} - Could not add!)") + } else { + summaryappend("(${tag} - Already added)") + } + } else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + echo "Tag added: ${tag} with sha: ${postcommitshacode}" + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${shacode})") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + sh("git remote -v") + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + sh("git remote -v") + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def boolean isalreadyreleased(buildmeta, tag) { + def boolean alreadyreleased = buildmeta.releaseinfo.released + if (alreadyreleased) { + echo "Component is already marked as released: ${alreadyreleased}" + } else { + String commitshacode = releasegitgetshacode(tag) + if (commitshacode) { + echo "Tag: ${tag} already exists in this repo therefore marking as already released" + alreadyreleased = true + } else { + alreadyreleased = false + } + } + return alreadyreleased +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { + summaryappend(" Git operations: ") + def String buildstamptag = buildmeta.releasetag() + + dir(releaserepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) + + def String postcommitshacode + def boolean changed = false + if (alreadyreleased) { + summaryappend("Already-released,") + postcommitshacode = releasegitgetshacode(buildstamptag) + echo "Previous release sha code: ${postcommitshacode}" + } else { + releasegitadd() + + changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamptag) + } else { + releasegitreset() + summaryappend("No-commit,") + } + + postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + } + + ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } + dir(sourcerepo) { + def String srcbuildstamptag = buildmeta.globalreleasetag() + def String buildsha = buildmeta.gitinfo.gitrevision + + if (buildsha) { + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } else { + summaryappend(",No Src ") + } + } +} + +def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { + def buildmeta = buildinfo.metainfo + def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) + def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) + + checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) + + if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + } + + summaryappend("\n") + repoandtagsappend("\n") +} + +def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { + def int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + versioncomponent += "${key}\n" + } + echo versioncomponent + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) + def String versionuniquecomponent = '' + def String versionwithpathscomponent = '' + for (key in componentReleaseInfoVersionUnique.keySet()) { + versionuniquecomponent += "${key}\n" + versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" + } + echo 'Version unique components:' + echo versionuniquecomponent + + echo 'Version unique components with paths:' + echo versionwithpathscomponent + + if (DO_INTEGRITY_CHECK == "true") { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" + } + } + + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { + error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" + } + } + + def toplevelid = buildmeta.componentidentifier() + // For each component + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + echo "Releasing component: ${buildinfo.metainfo.componentidentifier()}" + if (buildinfo.metainfo.componentidentifier() != '___' && buildinfo.metainfo.componentidentifier() != '_') { + if (toplevelid != key) { + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + } + } + } + // Do the top level last + def toplevelbuildinfo = componentReleaseInfo[toplevelid] + releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..f565706 --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo) { + if (releaserepo?.trim()) { + vc.checkRepoExists(releaserepo) + } + meta.releaseinfo.set(releaserepo) +} + +return this; diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..82af511 --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + BUILD_ARTIFACTS = 'build' + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) + importoptions.setincrementversionnumber(false) + importoptions.setfilter('meta/**') + importoptions.setappendtoparentmeta(false) + buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + // Place the top level build stamp on all + OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/remoteaccess/ftp.groovy b/remoteaccess/ftp.groovy new file mode 100644 index 0000000..25b9143 --- /dev/null +++ b/remoteaccess/ftp.groovy @@ -0,0 +1,32 @@ +import org.apache.commons.net.ftp.FTPClient + +class FTPSiteInfo { + String url + String username + String password + String workingdirectory + + FTPSiteInfo(String url, String username, String password, String workingdirectory) { + this.url = url + this.username = username + this.password = password + this.workingdirectory = workingdirectory + } +} + +FTPSiteInfo createFTPSiteInfo(String url, String username, String password, String workingdirectory) { + FTPSiteInfo ftpsiteinfo = new FTPSiteInfo(url, username, password, workingdirectory) + return ftpsiteinfo +} + +FTPClient createFTPClient(FTPSiteInfo ftpsiteinfo) { + FTPClient ftpclient = new FTPClient() + ftpclient.connect(ftpsiteinfo.url) + ftpclient.enterLocalPassiveMode() + ftpclient.login(ftpsiteinfo.username, ftpsiteinfo.password) + ftpclient.changeWorkingDirectory(ftpsiteinfo.workingdirectory) + + return ftpclient +} + +return this; diff --git a/remoteaccess/hostedftp.groovy b/remoteaccess/hostedftp.groovy new file mode 100644 index 0000000..cf62d32 --- /dev/null +++ b/remoteaccess/hostedftp.groovy @@ -0,0 +1,28 @@ +ftpremote = load("${autobuilder.commonscripts()}/remoteaccess/ftp.groovy") + +def getHostedFTPSiteInfo() { + hostedftpsiteinfo = + ftpremote.createFTPSiteInfo("ftp.hostedftp.com", + "ftp@globalinkjetsystems.com", + "ramshorn42", + "/Customers/") + return hostedftpsiteinfo +} + +def getCustomerList() { + def List customerfilenames = new ArrayList() + def List customers = new ArrayList() + + def ftpsiteinfo = getHostedFTPSiteInfo() + def ftpclient = ftpremote.createFTPClient(ftpsiteinfo) + + customerfilenames = ftpclient.listNames() + for(customer in customerfilenames) { + def String converted = new String(customer.getBytes("ISO-8859-1"), "UTF-8") + customers.add(converted) + } + ftpclient.disconnect() + return customers +} + +return this; \ No newline at end of file diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy new file mode 100644 index 0000000..ae91cb0 --- /dev/null +++ b/staticanalysis/ccm.groovy @@ -0,0 +1,85 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + CCMURL = 'git@hydra:tools/ccm.git' + CCMBRANCH = 'origin/master' + CCMDIR = "${WORKSPACE}/ccm" + configurationhelper.gitCheckoutAndRegister(meta, 'ccm', CCMURL, CCMBRANCH, CCMDIR) + CCM = "${CCMDIR}/bin/CCM.exe" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEDIR = "${WORKSPACE}/source" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) + } + } + } + stage("Clean exports") { + steps { + script { + RESULTS = 'results' + cleanfolders = [ + RESULTS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Static analysis') { + steps { + script { + def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" + writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) + echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" + + dir(RESULTS) { + echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" + if (isUnix()) { + sh """#!/bin/bash + ${CCM}" "${ccmconfigurationfile}" > ccm.xml + """ + } else { + echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" + bat """ + ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml + """ + } + } + recordIssues(tools: [ + ccm(pattern: "**/ccm.xml", reportEncoding: 'UTF-8'), + ]) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/staticanalysis/cpd.groovy b/staticanalysis/cpd.groovy new file mode 100644 index 0000000..c04649c --- /dev/null +++ b/staticanalysis/cpd.groovy @@ -0,0 +1,152 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHONBRANCH = 'origin/3.6.3_linux' + } else { + PYTHONBRANCH = 'origin/3.6.3_windows' + } + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + + JAVAURL = 'git@hydra:tools/openjdk.git' + if (isUnix()) { + JAVABRANCH = 'origin/openjdk-11.0.2_linux-x64_bin' + } else { + JAVABRANCH = 'origin/openjdk-11.0.2_windows-x64_bin' + } + JAVADIR = "${WORKSPACE}/java" + configurationhelper.gitCheckoutAndRegister(meta, 'java', JAVAURL, JAVABRANCH, JAVADIR) + JAVA = "${JAVADIR}/ver/bin" + if (isUnix()) { + env.PATH = JAVA + ':' + PATH + } else { + env.PATH = JAVA + ';' + PATH + } + + PPDURL = 'git@hydra:tools/pmd.git' + PPDBRANCH = 'origin/master' + PPDDIR = "${WORKSPACE}/ppd" + configurationhelper.gitCheckoutAndRegister(meta, 'ppd', PPDURL, PPDBRANCH, PPDDIR) + PPD = "${PPDDIR}/bin" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEREVISION = "${SOURCE_REVISION}" + SOURCEDIR = "${WORKSPACE}/source" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + SOURCE_PATH = "${SOURCEDIR}" + SRC = "${SOURCE_PATH}" + if (SOURCE_OFFSET_PATH) { + SRC += "/${SOURCE_OFFSET_PATH}" + } + } + } + } + stage("Clean exports") { + steps { + script { + STATICANALYSIS = 'staticanalysis' + cleanfolders = [ + STATICANALYSIS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.generate(meta, SOURCE_REVISION, SOURCE_PATH) + } + } + } + stage('Static analysis') { + steps { + script { + def PPDFOLDER = "${STATICANALYSIS}/ppd" + def EXCLUDE_PARAM = '' + if (EXCLUDE_PATHS) { + EXCLUDE_PARAM = '--exclude "' + def exclude_path_list = EXCLUDE_PATHS.split(',') + for (exclude_path in exclude_path_list) { + EXCLUDE_PARAM += "${SRC}/${exclude_path.trim()}," + } + EXCLUDE_PARAM += '"' + } + if (isUnix()) { + sh """#!/bin/bash + set -ev + + mkdir -p "${PPDFOLDER}" + export PMD_JAVA_OPTS=-Xmx65536m + "${PPD}/run.sh" cpd --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" + """ + } else { + bat """ + mkdir -p "${PPDFOLDER}" + set PMD_JAVA_OPTS=-Xmx2048m + "${PPD}/cpd.bat" --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" + """ + } + recordIssues(tools: [ + cpd(pattern: "**/cpd.xml", reportEncoding: 'UTF-8'), + ]) + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + ] + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] + def artifactlist = meta.generateartifacts(itemsForRelease) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy new file mode 100644 index 0000000..d959770 --- /dev/null +++ b/staticanalysis/cppcheck.groovy @@ -0,0 +1,114 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + PYTHONBRANCH = 'origin/3.6.3_windows' + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEREVISION = "${SOURCE_REVISION}" + SOURCEDIR = "${WORKSPACE}/source" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + SOURCE_PATH = "${SOURCEDIR}" + env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" + } + } + } + stage("Clean exports") { + steps { + script { + STATICANALYSIS = 'staticanalysis' + cleanfolders = [ + STATICANALYSIS, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.generate(meta, SOURCE_REVISION, SOURCE_PATH) + } + } + } + stage('Static analysis') { + steps { + script { + def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" + dir(CPPCHECKFOLDER) { + echo "Executing: ${CPPCHECK} --xml --xml-version=2 ${SRC} 2> cppcheck.xml" + if (isUnix()) { + sh """#!/bin/bash + set -e + + "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml + """ + } else { + bat """ + "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml + """ + } + } + recordIssues(tools: [ + cppCheck(pattern: "**/cppcheck.xml", reportEncoding: 'UTF-8'), + ]) + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + ] + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] + def artifactlist = meta.generateartifacts(itemsForRelease) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/test/testethernetproduct.groovy b/test/testethernetproduct.groovy new file mode 100644 index 0000000..5e2ce53 --- /dev/null +++ b/test/testethernetproduct.groovy @@ -0,0 +1,204 @@ +import jenkins.model.Jenkins + +pipeline { + agent { label AGENT_LABEL } + stages { + stage ("Initial Setup") { + steps{ + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage ("Checkout test environment") { + steps { + script { + toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") + + def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' + def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' + def TEST_ENVIRONMENT_REVISION = 'origin/master' + + def environmentoptions = toolsimporter.getNewToolOptions() + environmentoptions.toolName('Ethernet product test environment') + environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) + + TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" + def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") + TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] + echo "Selected environment:\n${TEST_ENVIRONMENT}" + } + } + } + stage ('Setup') { + agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } + stages { + stage ("Checkout dependencies") { + steps { + script { + def PYTHON_URL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHON_REVISION = 'origin/3.6.3_linux' + } + else { + PYTHON_REVISION = 'origin/3.8.10_windows_for_test' + } + PYTHONDIR = "${WORKSPACE}/python" + + def pythonimportoptions = toolsimporter.getNewToolOptions() + pythonimportoptions.toolName('Python') + pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) + toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + + def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' + TEST_SUITE_DIR = 'TestSuiteDir' + env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" + + def regressiontestimportoptions = toolsimporter.getNewToolOptions() + regressiontestimportoptions.toolName('Tests') + regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) + + env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" + + def resultimportoptions = toolsimporter.getNewToolOptions() + resultimportoptions.toolName('Test history') + resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) + toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) + + def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' + env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" + + def coreimportoptions = toolsimporter.getNewToolOptions() + coreimportoptions.toolName('Test core') + coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) + toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) + + def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' + def FTDI_SOURCE_REVISION = 'origin/Core-98' + FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" + + def ftdiimportoptions = toolsimporter.getNewToolOptions() + ftdiimportoptions.toolName('FTDI relay driver') + ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) + toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) + + def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' + HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" + + def resultprocessingoptions = toolsimporter.getNewToolOptions() + resultprocessingoptions.toolName('History processing') + resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) + toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) + } + } + } + stage ("Import application to test") { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + APPLICATION_UNDER_TEST = 'application_under_test' + env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" + + def applicationimportoptions = importer.getNewImportOptions() + applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) + applicationimportoptions.setfilter('app_package/**,control_package/**') + applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) + + meta.productinfo.set(applicationmeta.productinfo) + meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" + meta.gitinfo.set(applicationmeta.gitinfo) + meta.buildstampinfo.set(applicationmeta.buildstampinfo) + } + } + } + stage("Clean") { + steps { + script { + LOGGING_ROOT = 'Logging' + env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" + cleanfolders = [ + GIS_TEST_OUTPUT_LOGGING_ROOT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage ('Test') { + steps { + script { + lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { + if (POWER_CYCLE_HARDWARE == 'true') { + dir(FTDI_SOURCE_DIR) { + bat """ + "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True + """ + } + } + if (UPDATE_HARDWARE == 'true') { + dir(GIS_TEST_SUITE_CORE) { + bat """ + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" + """ + rc = bat (script:""" + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" + """, + returnStatus: true) + if (rc == 1) { + unstable('FAILED while inspecting state of hardware') + } + } + } + if (DO_TEST == 'true') { + rc = bat(script:""" + ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED while running tests') + } + } + } + } + } + } + stage("Archive Test History") { + steps{ + script { + dir(HISTORY_FOLDER) { + rc = bat (script:""" + "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED to process history') + } + } + } + } + } + stage("Archive test artifacts"){ + steps{ + script { + def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + } + } +} + diff --git a/textutils/subst.groovy b/textutils/subst.groovy new file mode 100644 index 0000000..c0207a3 --- /dev/null +++ b/textutils/subst.groovy @@ -0,0 +1,8 @@ + +def substitute(String filepath, pattern, String subst) { + fileText = readFile(file: filepath, encoding: 'UTF-8') + def converted = fileText.replaceAll(pattern, subst) + writeFile(file: filepath, text: converted, encoding: 'UTF-8') +} + +return this; \ No newline at end of file diff --git a/tool configurations/vs2017install/buildmachines.vsconfig b/tool configurations/vs2017install/buildmachines.vsconfig new file mode 100644 index 0000000..ea618a3 --- /dev/null +++ b/tool configurations/vs2017install/buildmachines.vsconfig @@ -0,0 +1,25 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Workload.ManagedDesktop", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.Workload.NetWeb", + "Microsoft.VisualStudio.Workload.NetCoreTools", + "microsoft.net.componentgroup.targetingpacks.common", + "microsoft.componentgroup.blend", + "microsoft.visualstudio.component.entityframework", + "microsoft.visualstudio.component.debugger.justintime", + "microsoft.visualstudio.component.wcf.tooling", + "microsoft.visualstudio.component.vc.diagnostictools", + "microsoft.visualstudio.component.vc.cmake.project", + "microsoft.visualstudio.component.vc.testadapterforboosttest", + "microsoft.visualstudio.component.vc.testadapterforgoogletest", + "microsoft.visualstudio.component.vc.atlmfc", + "microsoft.visualstudio.component.appinsights.tools", + "microsoft.visualstudio.component.diagnostictools", + "microsoft.visualstudio.componentgroup.web.cloudtools", + "microsoft.visualstudio.component.aspnet45", + "microsoft.visualstudio.component.webdeploy", + "microsoft.visualstudio.component.nuget.buildtools" + ] +} \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy new file mode 100644 index 0000000..a22054b --- /dev/null +++ b/versioncontrol/git.groovy @@ -0,0 +1,377 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") + +def boolean isShallowClonedRepo(String gitrootpath) { + def boolean isshallowclone = true + + dir( gitrootpath) { + if (isUnix()) { + isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() + } + else { + isshallowclone = powershell(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() + } + echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" + } + return isshallowclone +} + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git rev-parse HEAD') + } +} + +ArrayList gettaglist(String workingSetRootPath) { + ArrayList taglist = new ArrayList() + dir(workingSetRootPath) { + def tagliststring = clh.getCommandOutput('git tag --sort=-version:refname') + taglist = tagliststring.split('\\n') + } + return taglist +} + +@NonCPS +def ArrayList sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return sparsecheckoutpaths +} + +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } else { + def reporootname = uniquereporootname(gitURL) + reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootsubfolder}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootsubfolder}" +/* + dir(reporootsubfolder) { + if (isUnix()) { + sh """ + git status + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + powershell """ + git status + if (\$lastexitcode -ge 1) { exit 1 } + """ + } + } +*/ + } + } + return reporootsubfolder +} + +def checkgitlfsinstall() { + if (isUnix()) { + sh """ + git lfs install + if [ \$? -ne 0 ]; then exit 1; fi + """ + + } else { + powershell """ + git lfs install + if (\$lastexitcode -ge 1) { exit 1 } + """ + } +} + +//No credentials specified +//Cloning the remote Git repository +//Cloning repository git@hydra:print-server-releases/print_server_whole.git +//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 +//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git +//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// No valid HEAD. Skipping the resetting +// > git clean -fdx # timeout=10 +// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_7" +// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 + + +// > git rev-parse --is-inside-work-tree # timeout=10 +// Fetching changes from the remote Git repository +// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// Resetting working tree +// > git reset --hard # timeout=10 +// > git clean -fdx # timeout=10 +// Using shallow fetch with depth 1 +// Fetching upstream changes from git@hydra:releases/print-server-installer.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 +// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 +// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git read-tree -mu HEAD # timeout=10 +// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_3" +// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 + +def String lfsinstallandreporoot(String gitURL) { + checkgitlfsinstall() + return checkouttoreporoot(gitURL) +} + +def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { + if (!sparseitemlist.isEmpty()) { + def itemstring = "Items in sparse item list:\n" + for (item in sparseitemlist) { + itemstring += item + "\n" + } + echo itemstring + } +} + +def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { + echo "Git: Checking out: URL: ${gitURL} Revision: ${revision} to: ${targetDir}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'PruneStaleBranch'], + ], + submoduleCfg: [], + ]) +} + +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + +def gitinit(String targetDir, String reporootpath) { + if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + echo "git init ${targetDir}" + if (isUnix()) { + sh """ + git init "${targetDir}" + """ + } else { + powershell """ + git init "${targetDir}" + """ + } + } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") + } +} + +def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { + def String sparseitemliststring = "" + if (!sparseitemlist.isEmpty()) { + for (item in sparseitemlist) { + sparseitemliststring += "${item}," + } + echo "git config lfs.fetchinclude ${sparseitemliststring}" + dir(targetDir) { + if (isUnix()) { + sh """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } else { + powershell """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } + } + } +} + +def gitsparserestore(String targetDir) { + echo "Clean out full fetch; Force a sparse fetch" + dir(targetDir) { + if (isUnix()) { + sh """ + git rm -rf . > /dev/null + git clean -fxd > /dev/null + git reset --hard > /dev/null + """ + } else { + powershell """ + git rm -rf . > \$null + git clean -fxd > \$null + git reset --hard > \$null + """ + } + } + } + +def List convertSparseItemsString(String itemstopublish) { + def List sparsepaths = [] + + boolean nonempty = itemstopublish?.trim() + if (nonempty) { + def String[] items = itemstopublish.trim().split(',') + def String allpaths = "" + for(String item : items) { + sparsepaths.add(item) + allpaths += "${item}\n" + } + echo "Selected item to publish:\n${allpaths}" + } + return sparsepaths +} + +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + echo "Performing alternative checkout" + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitinit(targetDir, reporootpath) + setlfsfetchinclude(targetDir, sparseitemlist) + } + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitsparserestore(targetDir) + } +} + +def checkRepoExists(String gitURL) { + def int result = 0 + if (isUnix()) { + result = sh( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + """, + returnStatus: true + ) + } else { + result = powershell( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + """, + returnStatus: true + ) + } + echo "git ls-remote return result: ${result}" + if (result > 0) { + error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } else { + echo "The release repo: ${gitURL} exists" + } +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } + else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +def stripgithistory(String rootfolder) { + echo "Stripping git history out of ${rootfolder}" + dir(rootfolder) { + if (isUnix()) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + find . -iname "*@tmp" | xargs -iXX rm -rf XX + """ + } else { + echo "Not supporting git stripping in Windows at the moment" + } + } +} + +return this diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..208a60f --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,23 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + def filetoconvert = convertdetail[0] + def pattern = convertdetail[1] + def replacement = convertdetail[2] + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" + echo "Using python: ${PYTHON} and script: ${stampscript}" + if (isUnix()) { + sh """ + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + powershell """ + & "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + } +} + +return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..5e974e2 --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,180 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} + +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" + return newrevision +} + +def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl + + def boolean isShallowCloned = gh.isShallowClonedRepo(gitrootpath) + if (isShallowCloned) { + error("Cannot generate a version number from a shallow cloned repo at ${gitrootpath}") + } + + dir( gitrootpath) { + def int changecount = 0 + if (isUnix()) { + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() + } + else { + changecount = powershell(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() + } + echo "Change count: ${changecount}" + def int mvnp1 = maxversionnumber() + 1 + basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber + adjustedchangecount = incrementrevision(changecount, basebuildnumber) + echo "Adjusted change count: ${adjustedchangecount}" + if (isUnix()) { + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + } + else { + gitrevision = powershell(returnStdout: true, script: "git rev-parse HEAD").trim() + } + echo "Revision: ${gitrevision}" + if (isUnix()) { + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + } + else { + remoteinfo = powershell(returnStdout: true, script: "git remote show origin").trim() + } + echo "Remote info: ${remoteinfo}" + if (isUnix()) { + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + else { + remoteurl = powershell(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + echo "Remote URL: ${remoteurl}" + } + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] +} + +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + + def defaultversionprefix = "0.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } + + def mastertypeversionprefix = "0.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + + def splits2 = sourcebranch.split('\\.') + if (splits2.size() < 1) { + def thirdpartyversionprefix = "0.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + + for (split in splits2) { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + } + + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + if (integerstring.length() > 6) { + integerstring = integerstring.substring(0, 5) + } + int integer = integerstring.toInteger() + integer = integer % (maxversionnumber() + 1) + integers.add(integer) + } + + def versionprefix = "" + if (integers.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { + break + } + } + if (integernumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." + } + } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} + +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) + + def versionprefix = getVersionPrefix(sourcerevision) + + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" + + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) + + return buildstamp +} + +String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { + generate(meta, sourcerevision, sourcepath, basebuildnumber) + meta.buildstampinfo.override(pregeneratedbuildstamp) + + return pregeneratedbuildstamp +} + +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { + def int bbn = basebuildnumber.trim().toInteger() + return generate(meta, sourcerevision, sourcepath, bbn) +} + +String copyfrom(meta, othermeta) { + // Deep copy + meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) + meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) + + return meta.buildstampinfo.buildstamp +} + +return this; \ No newline at end of file diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..9243aa4 --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,38 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting instances of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + with open(filetoconvert,'r') as c: + fileread = c.read() + + count = 0 + with open(filetoconvert, 'w') as f: + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) + f.write(converted) + + if count == 0: + return 1 + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From bfd07dcb927fee80343eae481aee218d1fa4718c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 14:57:12 +0000 Subject: [PATCH 03/73] CORE-594 - Scripts as a starting point from MCS --- build/build_all.groovy | 225 +++++++++++++++++++++++++ compile/compile.groovy | 173 +++++++++++++++++++ documentation/release_note.groovy | 120 +++++++++++++ static_analysis/static_analysis.groovy | 70 ++++++++ subtrees.txt | 2 + 5 files changed, 590 insertions(+) create mode 100644 build/build_all.groovy create mode 100644 compile/compile.groovy create mode 100644 documentation/release_note.groovy create mode 100644 static_analysis/static_analysis.groovy create mode 100644 subtrees.txt diff --git a/build/build_all.groovy b/build/build_all.groovy new file mode 100644 index 0000000..b045248 --- /dev/null +++ b/build/build_all.groovy @@ -0,0 +1,225 @@ +def selectbuild(buildnumber) { + return "${buildnumber}" +} + +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + autobuilder.setBuildType(meta, (params.FULLBUILD != null) ? FULLBUILD.toBoolean() : false) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = 'git@hydra:gis/atlasservices.git' + SOURCEREVISION = "${SOURCE_REVISION}".trim() + SOURCEROOTDIR = 'atlasservices' + SOURCEDIR = "${WORKSPACE}/atlasservices" + configurationinfo = configurationhelper.gitCheckoutAndRegister(meta, SOURCEROOTDIR, SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + sourcesha = configurationinfo.configurationSHA + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.generate(meta, SOURCEREVISION, SOURCEDIR) + } + } + } + stage('Compile') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + buildstamp = meta.buildstampinfo.buildstamp + compile_job = build job: MCS_COMPILE_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + string(name: 'PRINT_SERVER_ARTIFACT_LOCATION', value: PRINT_SERVER_ARTIFACT_LOCATION), + string(name: 'PRINT_SERVER_BUILD_JOB', value: PRINT_SERVER_BUILD_JOB), + string(name: 'PRINT_SERVER_BUILD_SELECTOR', value: PRINT_SERVER_BUILD_SELECTOR), + string(name: 'PRINT_SERVER_RELEASE_REPO', value: PRINT_SERVER_RELEASE_REPO), + string(name: 'PRINT_SERVER_RELEASE_REVISION', value: PRINT_SERVER_RELEASE_REVISION), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + stage('Post compile') { + failFast true + parallel { + stage('Core install') { + steps { + script { + installer_job = build job: MCS_INSTALLER_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + string(name: 'MCS_COMPILE_JOB', value: compile_job.getProjectName()), + string(name: 'MCS_COMPILE_JOB_SELECTOR', value: selectbuild(compile_job.getNumber())), + string(name: 'LICENCEMANAGER_ARTIFACT_SOURCE', value: LICENCEMANAGER_ARTIFACT_SOURCE), + string(name: 'LICENCEMANAGER_BUILD_JOB', value: LICENCEMANAGER_BUILD_JOB), + string(name: 'LICENCEMANAGER_BUILD_SELECTOR', value: LICENCEMANAGER_BUILD_SELECTOR), + string(name: 'LICENCEMANAGER_RELEASE_REPO', value: LICENCEMANAGER_RELEASE_REPO), + string(name: 'LICENCEMANAGER_RELEASE_REVISION', value: LICENCEMANAGER_RELEASE_REVISION), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + stage('SDK') { + steps { + script { + if (meta.isFullBuild()) { + sdk_compile_job = build job: MCS_SDK_COMPILE_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + string(name: 'MCS_COMPILE_JOB', value: compile_job.getProjectName()), + string(name: 'MCS_COMPILE_JOB_SELECTOR', value: selectbuild(compile_job.getNumber())), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + script { + if (meta.isFullBuild()) { + sdk_installer_job = build job: MCS_SDK_INSTALLER_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + string(name: 'MCS_SDK_COMPILE_JOB', value: sdk_compile_job.getProjectName()), + string(name: 'MCS_SDK_COMPILE_BUILD_SELECTOR', value: selectbuild(sdk_compile_job.getNumber())), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + } + stage('Unit test') { + steps { + script { + unit_test_job = build job: MCS_UNIT_TEST_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + stage('Dev documentation') { + steps { + script { + if (meta.isFullBuild()) { + dev_documentation_job = build job: MCS_DEV_DOCUMENTATION_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + string(name: 'MCS_COMPILE_JOB', value: compile_job.getProjectName()), + string(name: 'MCS_COMPILE_JOB_SELECTOR', value: selectbuild(compile_job.getNumber())), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + } + stage('Release Documentation') { + steps { + script { + if (meta.isFullBuild()) { + release_notes_job = build job: RELEASE_NOTES_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] + } + } + } + } + } + } + stage('Accumulate artifacts') { + steps { + script { + artifacthelper = load("${autobuilder.commonscripts()}/artifacts/metaexport.groovy") + subJobs = [ + // Sub-job variable, Artifact folder Increment Version Number + [ compile_job, "mcs_binaries", false, ], + [ installer_job, "core_installer", false, ], + ] + if (meta.isFullBuild()) { + subJobs += [ + [ sdk_compile_job, "mcs_sdk_binaries", false, ], + [ sdk_installer_job, "sdk_installer", false, ], + [ dev_documentation_job, "dev_documentation", false, ], + [ release_notes_job, "release_notes", false, ], + ] + } + artifactFolders = artifacthelper.combineSubjobArtifacts(meta, importer, subJobs) + } + } + } + stage('Build integrity check') { + steps { + script { + def integrity = load("${autobuilder.commonscripts()}/build/integrity.groovy") + def results = integrity.check(meta) + if (!results.good) { + unstable("Build integrity check failed: \n ${results.comment}") + } + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + 'meta/**', + ] + def publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts([], artifactFolders) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() + } + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() + } + } + } +} diff --git a/compile/compile.groovy b/compile/compile.groovy new file mode 100644 index 0000000..d5508f9 --- /dev/null +++ b/compile/compile.groovy @@ -0,0 +1,173 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + PYTHONBRANCH = 'origin/3.6.3_windows' + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEURL = 'git@hydra:gis/atlasservices.git' + SOURCEREVISION = "${SOURCE_REVISION}" + SOURCEDIR = "${WORKSPACE}\\atlasservices" + configurationhelper.gitCheckoutAndRegister(meta, 'atlasservices', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + env.SOURCE_PATH = SOURCEDIR + env.JOBSCRIPTS = "${SOURCE_PATH}\\autobuild\\build" + env.JOBSCRIPTS_COMMON = "${SOURCE_PATH}\\autobuild\\common" + env.SRC = "${SOURCE_PATH}/src" + } + } + } + stage('Clean exports') { + steps { + script { + env.BUILDOUTPUT = "${WORKSPACE}/build" + env.PS_WHOLE_ARTIFACTS_ROOT = "${WORKSPACE}/ps_whole" + env.PSBIN = "${WORKSPACE}/ps_bin" + cleanfolders = [ + BUILDOUTPUT, + "${WORKSPACE}/SharedPackages", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.pregenerated(meta, SOURCEREVISION, SOURCEDIR, PARENT_BUILD_STAMP) + } + } + } + stage('Import Print server to adjust version number') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + // Alter the version number + def pswholeimportoptions = importer.getNewImportOptions() + pswholeimportoptions.source(PRINT_SERVER_ARTIFACT_LOCATION, + PRINT_SERVER_BUILD_JOB, + PRINT_SERVER_BUILD_SELECTOR, + PRINT_SERVER_RELEASE_REPO, + PRINT_SERVER_RELEASE_REVISION) + pswholeimportoptions.setfilter('meta/**') + pswholeimportoptions.setincrementversionnumber(false) + pswholeimportoptions.setappendtoparentmeta(false) + pswholemeta = importer.artifactimport(meta, pswholeimportoptions, PS_WHOLE_ARTIFACTS_ROOT) + + // Import the binaries from the related component + def psbinimportoptions = importer.getNewImportOptions() + psbinimportoptions.setsourceselect(PRINT_SERVER_ARTIFACT_LOCATION) + psbinimportoptions.setfilter('meta/**,bin/x64/Release/Engine - Render.dll,bin/x64/Release/Engine - Render.lib,bin/x64/Release/Engine - Render.pdb,bin/x64/Release/TBarCode11_x64.dll,bin/x64/Release/GIS Utility - Toolbox.dll,bin/x64/Release/GIS Utility - Toolbox.lib,bin/x64/Release/GIS Utility - Toolbox.pdb,bin/x64/Release/zlib1.dll,bin/x64/Release/zlib1.lib,bin/x64/Release/zlib1.pdb,bin/x64/Release/GIS Utility - TCP-IP Comms.dll,bin/x64/Release/GIS Utility - TCP-IP Comms.lib,bin/x64/Release/GIS Utility - TCP-IP Comms.pdb,bin/x64/Release/Dialog - Tree Node Editor.dll,bin/x64/Release/Dialog - Tree Node Editor.lib,bin/x64/Release/Dialog - Tree Node Editor.pdb') + psbinimportoptions.setincrementversionnumber(false) + psbinmeta = importer.artifactimport(meta, pswholemeta, PS_WHOLE_ARTIFACTS_ROOT, 'PRINT_SERVER', 'CORE_BINARIES', psbinimportoptions, PSBIN) + } + } + } + stage('Version stamp files') { + steps { + script { + env.BUILD_STAMP = meta.buildstampinfo.buildstamp + env.BUILD_DATE = meta.jenkinsbuildinfo.builddate + bat """ + call "${JOBSCRIPTS}\\versionFiles.bat" + if errorlevel 1 exit /b 1 + """ + } + } + } + stage('Compile application') { + steps { + script { + env.SRC_REVISION=SOURCE_REVISION + bat """ + call "${JOBSCRIPTS}\\autobuild.bat" + if errorlevel 1 exit /b 1 + """ + } + } + } + stage('Prepare exports') { + steps { + script { + bat """ + xcopy /i /e "${SRC}\\PackageSpecs\\*.nupkg" "${WORKSPACE}\\SharedPackages" + if errorlevel 1 exit /b 1 + """ + } + } + } + stage('Static analysis') { + steps { + script { + recordIssues(tools: [ + msBuild(enabledForFailure: true, reportEncoding: 'UTF-8'), + ]) + } + } + } + stage('Checksum artifacts') { + steps { + script { + def checksummable = [ + ] + checksumutils = load("${autobuilder.commonscripts()}/checksum/utils.groovy") + checksumutils.generatelist(meta, checksummable) + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + ] + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(['build/**','SharedPackages/**']) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/documentation/release_note.groovy b/documentation/release_note.groovy new file mode 100644 index 0000000..2241378 --- /dev/null +++ b/documentation/release_note.groovy @@ -0,0 +1,120 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHONBRANCH = "origin/3.6.3_linux" + } + else { + PYTHONBRANCH = "origin/3.6.3_windows" + } + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEURL = 'git@hydra:gis/atlasservices.git' + SOURCEREVISION = "${SOURCE_REVISION}".trim() + SOURCEDIR = "${WORKSPACE}/atlasservices" + configurationhelper.gitCheckoutAndRegister(meta, 'atlasservices', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + env.SOURCE_PATH = SOURCEDIR + } + } + } + stage('Clean exports') { + steps { + script { + env.RELEASE_NOTES = "release_notes" + cleanfolders = [ + RELEASE_NOTES, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.pregenerated(meta, SOURCEREVISION, SOURCEDIR, PARENT_BUILD_STAMP) + } + } + } + stage('Copy release notes') { + steps { + script { + dir("${SOURCEDIR}/releasenotes") { + fileOperations([fileCopyOperation(targetLocation: "${WORKSPACE}/${RELEASE_NOTES}", includes: '*.txt')]) + } + } + } + } + stage('Checksum artifacts') { + steps { + script { + def checksummable = [ + "${RELEASE_NOTES}/**", + ] + checksumutils = load("${autobuilder.commonscripts()}/checksum/utils.groovy") + checksumutils.generatelist(meta, checksummable) + } + } + } + stage('Classifying publishable artifacts') { + steps { + script { + def publishable = [ + "${RELEASE_NOTES}/**", + ] + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["${RELEASE_NOTES}/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/static_analysis/static_analysis.groovy b/static_analysis/static_analysis.groovy new file mode 100644 index 0000000..efa72e2 --- /dev/null +++ b/static_analysis/static_analysis.groovy @@ -0,0 +1,70 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + echo "Starting...." + AUTOBUILD_SCRIPTS = "${WORKSPACE}/autobuild" + env.COMMON_SCRIPTS = "${AUTOBUILD_SCRIPTS}/Common_scripts" + githelper = load("${COMMON_SCRIPTS}/versioncontrol/git.groovy") + configurationhelper = load("${COMMON_SCRIPTS}/configuration/configuration.groovy") + configurationhelper.registerPipelineScriptConfiguration() + } + } + } + stage('Clean workspace') { + when { + expression { + "${CLEAN_WORKSPACE}" == "true" + } + } + steps { + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + cleanWs() + unstash name: 'Clean workspace stash' + } + } + stage('Print server static analysis') { + parallel { + stage('Measure Print Server code complexity') { + steps { + build job: "${PRINT_SERVER_CCM_JOB}" + } + } + stage('Perform Print Server C++ static analysis') { + steps { + build job: "${PRINT_SERVER_CPP_STATIC_ANALYSIS_JOB}" + } + } + stage('Perform Print Server DRY static analysis') { + steps { + build job: "${PRINT_SERVER_DRY_ANALYSIS_JOB}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' +/* + step([$class: 'LogParserPublisher', + parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', + useProjectRule: true + ]) + */ + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/subtrees.txt b/subtrees.txt new file mode 100644 index 0000000..fbf1dfa --- /dev/null +++ b/subtrees.txt @@ -0,0 +1,2 @@ +Local folder: Remote repo: Revision: +Common_scripts git@hydra:jenkinsautobuild/Common_scripts.git (push) master From b814fc4d55e685d340472916b9b8939138b71e79 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 15:25:59 +0000 Subject: [PATCH 04/73] CORE-594 - Create build script for LNPU --- build/build_all.groovy | 119 ++++++----------------------------------- 1 file changed, 15 insertions(+), 104 deletions(-) diff --git a/build/build_all.groovy b/build/build_all.groovy index b045248..4cb6cc4 100644 --- a/build/build_all.groovy +++ b/build/build_all.groovy @@ -18,10 +18,10 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - SOURCEURL = 'git@hydra:gis/atlasservices.git' + SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}".trim() - SOURCEROOTDIR = 'atlasservices' - SOURCEDIR = "${WORKSPACE}/atlasservices" + SOURCEROOTDIR = 'lnpu' + SOURCEDIR = "${WORKSPACE}/lnpu" configurationinfo = configurationhelper.gitCheckoutAndRegister(meta, SOURCEROOTDIR, SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) sourcesha = configurationinfo.configurationSHA } @@ -41,114 +41,29 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") buildstamp = meta.buildstampinfo.buildstamp - compile_job = build job: MCS_COMPILE_JOB, + compile_job = build job: LNPU_COMPILE_JOB, parameters: [ string(name: 'PARENT_BUILD_STAMP', value: buildstamp), string(name: 'SOURCE_REVISION', value: sourcesha), - string(name: 'PRINT_SERVER_ARTIFACT_LOCATION', value: PRINT_SERVER_ARTIFACT_LOCATION), - string(name: 'PRINT_SERVER_BUILD_JOB', value: PRINT_SERVER_BUILD_JOB), - string(name: 'PRINT_SERVER_BUILD_SELECTOR', value: PRINT_SERVER_BUILD_SELECTOR), - string(name: 'PRINT_SERVER_RELEASE_REPO', value: PRINT_SERVER_RELEASE_REPO), - string(name: 'PRINT_SERVER_RELEASE_REVISION', value: PRINT_SERVER_RELEASE_REVISION), booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), ] } } } - stage('Post compile') { - failFast true - parallel { - stage('Core install') { - steps { - script { - installer_job = build job: MCS_INSTALLER_JOB, - parameters: [ - string(name: 'PARENT_BUILD_STAMP', value: buildstamp), - string(name: 'SOURCE_REVISION', value: sourcesha), - string(name: 'MCS_COMPILE_JOB', value: compile_job.getProjectName()), - string(name: 'MCS_COMPILE_JOB_SELECTOR', value: selectbuild(compile_job.getNumber())), - string(name: 'LICENCEMANAGER_ARTIFACT_SOURCE', value: LICENCEMANAGER_ARTIFACT_SOURCE), - string(name: 'LICENCEMANAGER_BUILD_JOB', value: LICENCEMANAGER_BUILD_JOB), - string(name: 'LICENCEMANAGER_BUILD_SELECTOR', value: LICENCEMANAGER_BUILD_SELECTOR), - string(name: 'LICENCEMANAGER_RELEASE_REPO', value: LICENCEMANAGER_RELEASE_REPO), - string(name: 'LICENCEMANAGER_RELEASE_REVISION', value: LICENCEMANAGER_RELEASE_REVISION), - booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), - ] - } + stage('Release Documentation') { + steps { + script { + if (meta.isFullBuild()) { + release_notes_job = build job: RELEASE_NOTES_JOB, + parameters: [ + string(name: 'PARENT_BUILD_STAMP', value: buildstamp), + string(name: 'SOURCE_REVISION', value: sourcesha), + booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), + ] } } - stage('SDK') { - steps { - script { - if (meta.isFullBuild()) { - sdk_compile_job = build job: MCS_SDK_COMPILE_JOB, - parameters: [ - string(name: 'PARENT_BUILD_STAMP', value: buildstamp), - string(name: 'SOURCE_REVISION', value: sourcesha), - string(name: 'MCS_COMPILE_JOB', value: compile_job.getProjectName()), - string(name: 'MCS_COMPILE_JOB_SELECTOR', value: selectbuild(compile_job.getNumber())), - booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), - ] - } - } - script { - if (meta.isFullBuild()) { - sdk_installer_job = build job: MCS_SDK_INSTALLER_JOB, - parameters: [ - string(name: 'PARENT_BUILD_STAMP', value: buildstamp), - string(name: 'SOURCE_REVISION', value: sourcesha), - string(name: 'MCS_SDK_COMPILE_JOB', value: sdk_compile_job.getProjectName()), - string(name: 'MCS_SDK_COMPILE_BUILD_SELECTOR', value: selectbuild(sdk_compile_job.getNumber())), - booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), - ] - } - } - } - } - stage('Unit test') { - steps { - script { - unit_test_job = build job: MCS_UNIT_TEST_JOB, - parameters: [ - string(name: 'PARENT_BUILD_STAMP', value: buildstamp), - string(name: 'SOURCE_REVISION', value: sourcesha), - booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), - ] - } - } - } - stage('Dev documentation') { - steps { - script { - if (meta.isFullBuild()) { - dev_documentation_job = build job: MCS_DEV_DOCUMENTATION_JOB, - parameters: [ - string(name: 'PARENT_BUILD_STAMP', value: buildstamp), - string(name: 'SOURCE_REVISION', value: sourcesha), - string(name: 'MCS_COMPILE_JOB', value: compile_job.getProjectName()), - string(name: 'MCS_COMPILE_JOB_SELECTOR', value: selectbuild(compile_job.getNumber())), - booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), - ] - } - } - } - } - stage('Release Documentation') { - steps { - script { - if (meta.isFullBuild()) { - release_notes_job = build job: RELEASE_NOTES_JOB, - parameters: [ - string(name: 'PARENT_BUILD_STAMP', value: buildstamp), - string(name: 'SOURCE_REVISION', value: sourcesha), - booleanParam(name: 'CLEAN_WORKSPACE', value: CLEAN_WORKSPACE), - ] - } - } - } - } } - } + } stage('Accumulate artifacts') { steps { script { @@ -156,13 +71,9 @@ pipeline { subJobs = [ // Sub-job variable, Artifact folder Increment Version Number [ compile_job, "mcs_binaries", false, ], - [ installer_job, "core_installer", false, ], ] if (meta.isFullBuild()) { subJobs += [ - [ sdk_compile_job, "mcs_sdk_binaries", false, ], - [ sdk_installer_job, "sdk_installer", false, ], - [ dev_documentation_job, "dev_documentation", false, ], [ release_notes_job, "release_notes", false, ], ] } From c5c46e434f5e7260feb563946ab02eff0c4434a9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 15:31:45 +0000 Subject: [PATCH 05/73] CORE-594 - Create compile script for LNPU --- compile/compile.groovy | 49 ++++-------------------------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index d5508f9..a00ffb5 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -24,13 +24,11 @@ pipeline { stage('Checkout source') { steps { script { - SOURCEURL = 'git@hydra:gis/atlasservices.git' + SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}" - SOURCEDIR = "${WORKSPACE}\\atlasservices" - configurationhelper.gitCheckoutAndRegister(meta, 'atlasservices', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + SOURCEDIR = "${WORKSPACE}\\lnpu" + configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) env.SOURCE_PATH = SOURCEDIR - env.JOBSCRIPTS = "${SOURCE_PATH}\\autobuild\\build" - env.JOBSCRIPTS_COMMON = "${SOURCE_PATH}\\autobuild\\common" env.SRC = "${SOURCE_PATH}/src" } } @@ -39,11 +37,8 @@ pipeline { steps { script { env.BUILDOUTPUT = "${WORKSPACE}/build" - env.PS_WHOLE_ARTIFACTS_ROOT = "${WORKSPACE}/ps_whole" - env.PSBIN = "${WORKSPACE}/ps_bin" cleanfolders = [ BUILDOUTPUT, - "${WORKSPACE}/SharedPackages", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -58,32 +53,6 @@ pipeline { } } } - stage('Import Print server to adjust version number') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - // Alter the version number - def pswholeimportoptions = importer.getNewImportOptions() - pswholeimportoptions.source(PRINT_SERVER_ARTIFACT_LOCATION, - PRINT_SERVER_BUILD_JOB, - PRINT_SERVER_BUILD_SELECTOR, - PRINT_SERVER_RELEASE_REPO, - PRINT_SERVER_RELEASE_REVISION) - pswholeimportoptions.setfilter('meta/**') - pswholeimportoptions.setincrementversionnumber(false) - pswholeimportoptions.setappendtoparentmeta(false) - pswholemeta = importer.artifactimport(meta, pswholeimportoptions, PS_WHOLE_ARTIFACTS_ROOT) - - // Import the binaries from the related component - def psbinimportoptions = importer.getNewImportOptions() - psbinimportoptions.setsourceselect(PRINT_SERVER_ARTIFACT_LOCATION) - psbinimportoptions.setfilter('meta/**,bin/x64/Release/Engine - Render.dll,bin/x64/Release/Engine - Render.lib,bin/x64/Release/Engine - Render.pdb,bin/x64/Release/TBarCode11_x64.dll,bin/x64/Release/GIS Utility - Toolbox.dll,bin/x64/Release/GIS Utility - Toolbox.lib,bin/x64/Release/GIS Utility - Toolbox.pdb,bin/x64/Release/zlib1.dll,bin/x64/Release/zlib1.lib,bin/x64/Release/zlib1.pdb,bin/x64/Release/GIS Utility - TCP-IP Comms.dll,bin/x64/Release/GIS Utility - TCP-IP Comms.lib,bin/x64/Release/GIS Utility - TCP-IP Comms.pdb,bin/x64/Release/Dialog - Tree Node Editor.dll,bin/x64/Release/Dialog - Tree Node Editor.lib,bin/x64/Release/Dialog - Tree Node Editor.pdb') - psbinimportoptions.setincrementversionnumber(false) - psbinmeta = importer.artifactimport(meta, pswholemeta, PS_WHOLE_ARTIFACTS_ROOT, 'PRINT_SERVER', 'CORE_BINARIES', psbinimportoptions, PSBIN) - } - } - } stage('Version stamp files') { steps { script { @@ -107,16 +76,6 @@ pipeline { } } } - stage('Prepare exports') { - steps { - script { - bat """ - xcopy /i /e "${SRC}\\PackageSpecs\\*.nupkg" "${WORKSPACE}\\SharedPackages" - if errorlevel 1 exit /b 1 - """ - } - } - } stage('Static analysis') { steps { script { @@ -149,7 +108,7 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(['build/**','SharedPackages/**']) + def artifactlist = meta.generateartifacts(['build/**']) artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") artifactarchiver.archive(meta, artifactlist) } From 10a4a8010bb2cc2413d3ebf4a3c1ed9096416cf4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 15:33:27 +0000 Subject: [PATCH 06/73] CORE-594 - Create relese note script for LNPU --- documentation/release_note.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/release_note.groovy b/documentation/release_note.groovy index 2241378..7c60009 100644 --- a/documentation/release_note.groovy +++ b/documentation/release_note.groovy @@ -34,10 +34,10 @@ pipeline { stage('Checkout source') { steps { script { - SOURCEURL = 'git@hydra:gis/atlasservices.git' + SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}".trim() - SOURCEDIR = "${WORKSPACE}/atlasservices" - configurationhelper.gitCheckoutAndRegister(meta, 'atlasservices', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + SOURCEDIR = "${WORKSPACE}/lnpu" + configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) env.SOURCE_PATH = SOURCEDIR } } From 98690c5bffad5d1bb23268ac4626a59c871dd632 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 16:45:25 +0000 Subject: [PATCH 07/73] CORE-584 - Compile script --- compile/compile.groovy | 78 +++++++++++++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 13 deletions(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index a00ffb5..dcf03bb 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -18,6 +18,13 @@ pipeline { PYTHONDIR = "${WORKSPACE}/python" configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) env.PYTHON = "${PYTHONDIR}/ver/python.exe" + + VSWHEREURL = 'git@hydra:tools/vswhere.git' + VSWHEREREVISION = "refs/tags/${VSWHERE_VERSION}" + VSWHEREDIR = "${WORKSPACE}/vswhere" + configurationhelper.gitCheckoutAndRegister(meta, 'VS where', VSWHEREURL, VSWHEREREVISION, VSWHEREDIR) + env.VSWHERE="${VSWHEREDIR}/ver/vswhere.exe" + echo "VSWHERE is here: ${VSWHERE}" } } } @@ -26,10 +33,9 @@ pipeline { script { SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}" - SOURCEDIR = "${WORKSPACE}\\lnpu" - configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - env.SOURCE_PATH = SOURCEDIR - env.SRC = "${SOURCE_PATH}/src" + SOURCE_PATH = "${WORKSPACE}\\lnpu" + configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCE_PATH, '', 0, false) + env.SRC = "${SOURCE_PATH}/Local Network Package Updater" } } } @@ -56,22 +62,68 @@ pipeline { stage('Version stamp files') { steps { script { - env.BUILD_STAMP = meta.buildstampinfo.buildstamp - env.BUILD_DATE = meta.jenkinsbuildinfo.builddate - bat """ - call "${JOBSCRIPTS}\\versionFiles.bat" - if errorlevel 1 exit /b 1 - """ + echo ("BUILD_STAMP is ${meta.buildstampinfo.buildstamp}") + echo ("BUILD_DATE is ${meta.jenkinsbuildinfo.builddate}") + convertdetails = [ + // File to convert Pattern to replace New string + [ "${SRC}/Local Network Package Updater/Properties/AssemblyInfo.cs", "2.30000.0.0", meta.buildstampinfo.buildstamp() ], + ] + versionnumbering = load("${autobuilder.commonscripts()}/versionnumbering/do.groovy") + versionnumbering.stamp(convertdetails) } } } stage('Compile application') { steps { script { - env.SRC_REVISION=SOURCE_REVISION bat """ - call "${JOBSCRIPTS}\\autobuild.bat" - if errorlevel 1 exit /b 1 + REM Load the visual studio environment + @echo on + + REM run VC VARS to set up environment + for /f "usebackq tokens=*" %%i in (`%VSWHERE% -version "[16.0,)" -products Microsoft.VisualStudio.Product.Professional -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + call "%%i%\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.29 + if errorlevel 1 exit /b 1 + ) + + set INCLUDE + + set VERBOSITY=/v:n + set MAXCPUCOUNT=/m + set NODEREUSE=/nr:False + + echo AUTOBUILD 1 + + REM Default to Release + set RELEASE_CONFIGURATION=Release + set RELEASE_LOCATION=Release + if "%DO_DEBUG%"=="true" ( + set RELEASE_CONFIGURATION=Debug + set RELEASE_LOCATION=Debug + ) + + set MSBuildEmitSolution=1 + + echo Building variants: %DO_DEBUG% %RELEASE_CONFIGURATION% %RELEASE_LOCATION% + + @echo on + + for /f "usebackq tokens=*" %%M in (`%VSWHERE% -version "[16.0,)" -products Microsoft.VisualStudio.Product.Professional -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe`) do ( + call :mkbuildsubroutine "%%M" + if errorlevel 1 EXIT /b 1 + ) + if errorlevel 1 EXIT /b 1 + + EXIT /b 0 + + :mkbuildsubroutine + echo mkbuildsubroutine = %1 + + REM Perform a clean build of main product 64bit + %1 "%SRC%\Local Network Package Updater.sln" /t:Rebuild /p:Configuration=%RELEASE_CONFIGURATION% /p:Platform=x64 %MAXCPUCOUNT% %NODEREUSE% %VERBOSITY% + if errorlevel 1 EXIT /b 1 + + EXIT /b 0 """ } } From e9e01a00ed0e70c0938135c0943b7fd17164bf4a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 16:51:31 +0000 Subject: [PATCH 08/73] CORE-584 - Compile script --- compile/compile.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index dcf03bb..46988d8 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -33,9 +33,9 @@ pipeline { script { SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}" - SOURCE_PATH = "${WORKSPACE}\\lnpu" + SOURCE_PATH = 'lnpu' configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCE_PATH, '', 0, false) - env.SRC = "${SOURCE_PATH}/Local Network Package Updater" + env.SRC = "${SOURCE_PATH}\\Local Network Package Updater" } } } @@ -82,7 +82,7 @@ pipeline { REM run VC VARS to set up environment for /f "usebackq tokens=*" %%i in (`%VSWHERE% -version "[16.0,)" -products Microsoft.VisualStudio.Product.Professional -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( - call "%%i%\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.29 + call "%%i%\\VC\\Auxiliary\\Build\\vcvars64.bat" -vcvars_ver=14.29 if errorlevel 1 exit /b 1 ) @@ -120,7 +120,7 @@ pipeline { echo mkbuildsubroutine = %1 REM Perform a clean build of main product 64bit - %1 "%SRC%\Local Network Package Updater.sln" /t:Rebuild /p:Configuration=%RELEASE_CONFIGURATION% /p:Platform=x64 %MAXCPUCOUNT% %NODEREUSE% %VERBOSITY% + %1 "%SRC%\\Local Network Package Updater.sln" /t:Rebuild /p:Configuration=%RELEASE_CONFIGURATION% /p:Platform=x64 %MAXCPUCOUNT% %NODEREUSE% %VERBOSITY% if errorlevel 1 EXIT /b 1 EXIT /b 0 From 48990a5f8c849a51099cc0e724a4649be6041675 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 16:52:25 +0000 Subject: [PATCH 09/73] CORE-584 - Compile script --- compile/compile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index 46988d8..b9b9733 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -108,7 +108,7 @@ pipeline { @echo on - for /f "usebackq tokens=*" %%M in (`%VSWHERE% -version "[16.0,)" -products Microsoft.VisualStudio.Product.Professional -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe`) do ( + for /f "usebackq tokens=*" %%M in (`%VSWHERE% -version "[16.0,)" -products Microsoft.VisualStudio.Product.Professional -requires Microsoft.Component.MSBuild -find MSBuild\\**\\Bin\\MSBuild.exe`) do ( call :mkbuildsubroutine "%%M" if errorlevel 1 EXIT /b 1 ) From fc1da1eb0a8a37bb26d3650bf1f53cd19b4a0303 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 16:55:37 +0000 Subject: [PATCH 10/73] CORE-584 - Compile script --- compile/compile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index b9b9733..c161b18 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -55,7 +55,7 @@ pipeline { steps { script { version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.pregenerated(meta, SOURCEREVISION, SOURCEDIR, PARENT_BUILD_STAMP) + version.pregenerated(meta, SOURCEREVISION, SOURCE_PATH, PARENT_BUILD_STAMP) } } } From 4742e1cd11c52aaa16e7f4d3173776b7a5ce5a5b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 16:57:06 +0000 Subject: [PATCH 11/73] CORE-584 - Compile script --- compile/compile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index c161b18..fcd3cb6 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -66,7 +66,7 @@ pipeline { echo ("BUILD_DATE is ${meta.jenkinsbuildinfo.builddate}") convertdetails = [ // File to convert Pattern to replace New string - [ "${SRC}/Local Network Package Updater/Properties/AssemblyInfo.cs", "2.30000.0.0", meta.buildstampinfo.buildstamp() ], + [ "${SRC}/Local Network Package Updater/Properties/AssemblyInfo.cs", "2.30000.0.0", meta.buildstampinfo.buildstamp ], ] versionnumbering = load("${autobuilder.commonscripts()}/versionnumbering/do.groovy") versionnumbering.stamp(convertdetails) From c83236a596ca29dd4fdad6eeac5abf67f56f9c01 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 17:00:20 +0000 Subject: [PATCH 12/73] CORE-584 - Compile script --- compile/compile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index fcd3cb6..a3a2f6f 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -120,7 +120,7 @@ pipeline { echo mkbuildsubroutine = %1 REM Perform a clean build of main product 64bit - %1 "%SRC%\\Local Network Package Updater.sln" /t:Rebuild /p:Configuration=%RELEASE_CONFIGURATION% /p:Platform=x64 %MAXCPUCOUNT% %NODEREUSE% %VERBOSITY% + %1 "%SRC%\\Local Network Package Updater.sln" /t:Rebuild /p:Configuration=%RELEASE_CONFIGURATION% /p:Platform="Any CPU" %MAXCPUCOUNT% %NODEREUSE% %VERBOSITY% if errorlevel 1 EXIT /b 1 EXIT /b 0 From a70bdadea746008d793ddbc2d811446ab746a5cd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 17:26:17 +0000 Subject: [PATCH 13/73] CORE-584 - Compile script --- compile/compile.groovy | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index a3a2f6f..cedf888 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -86,25 +86,19 @@ pipeline { if errorlevel 1 exit /b 1 ) - set INCLUDE - set VERBOSITY=/v:n set MAXCPUCOUNT=/m set NODEREUSE=/nr:False - echo AUTOBUILD 1 - REM Default to Release set RELEASE_CONFIGURATION=Release - set RELEASE_LOCATION=Release if "%DO_DEBUG%"=="true" ( set RELEASE_CONFIGURATION=Debug - set RELEASE_LOCATION=Debug ) set MSBuildEmitSolution=1 - echo Building variants: %DO_DEBUG% %RELEASE_CONFIGURATION% %RELEASE_LOCATION% + echo Building variants: %DO_DEBUG% %RELEASE_CONFIGURATION% @echo on From f385118dc5c3fce53d1e0fcab5c6ad710f169ec5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Jan 2022 17:33:02 +0000 Subject: [PATCH 14/73] CORE-594 - Correct artifact output folder --- build/build_all.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build_all.groovy b/build/build_all.groovy index 4cb6cc4..8f02dd0 100644 --- a/build/build_all.groovy +++ b/build/build_all.groovy @@ -70,7 +70,7 @@ pipeline { artifacthelper = load("${autobuilder.commonscripts()}/artifacts/metaexport.groovy") subJobs = [ // Sub-job variable, Artifact folder Increment Version Number - [ compile_job, "mcs_binaries", false, ], + [ compile_job, "lnpu_binaries", false, ], ] if (meta.isFullBuild()) { subJobs += [ From 43a36d706e3809da7a68ab99ecfcf3a17558617c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 09:45:28 +0000 Subject: [PATCH 15/73] CORE-594 - Start with the print server installer script --- install/install.groovy | 351 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 install/install.groovy diff --git a/install/install.groovy b/install/install.groovy new file mode 100644 index 0000000..346d06c --- /dev/null +++ b/install/install.groovy @@ -0,0 +1,351 @@ +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + + if (INSTALLSHIELD2020_CMD == "" || DOTNET_FRAMEWORK64_PATH == "" || WINDOWS_DRIVER_KIT_8_1 == "" || WINDOWS_DRIVER_KIT_10 == "") { + echo "Either INSTALLSHIELD2020_CMD, DOTNET_FRAMEWORK64_PATH, WINDOWS_DRIVER_KIT_8_1 or WINDOWS_DRIVER_KIT_10 environment variables have not been defined" + currentBuild.result = 'FAILED' + exit(1) + } + installerlist = env.INSTALLERS_TO_BUILD + if (installerlist != null) { + installerlist = installerlist.trim() + } else { + installerlist = '' + } + toBuild = [] + if (installerlist) { + toBuildUntrimmed = installerlist.split(',') + toBuildUntrimmed.each { + toBuild.add(it.trim()) + } + } + } + echo "Installer list: ${installerlist}" + } + } + stage('Nothing to build check') { + when { + expression { + !toBuild + } + } + steps { + echo 'No installers to build' + script { + currentBuild.result = 'UNSTABLE' + } + } + } + stage('Checkout dependencies') { + when { + expression { + toBuild + } + } + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + PYTHONBRANCH = 'origin/3.6.3_windows' + PYTHONDIR = "${WORKSPACE}/python" + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + + SUPPORTURL = 'git@hydra:gis/gissupport.git' + SUPPORTBRANCH = SUPPORT_REVISION.trim() + SUPPORTDIR = "${WORKSPACE}\\gissupport" + configurationhelper.gitCheckoutAndRegister(meta, 'gissupport', SUPPORTURL, SUPPORTBRANCH, SUPPORTDIR) + env.INST_SUPPORT = SUPPORTDIR + + EULAURL = 'git@hydra:gis/giseula.git' + EULABRANCH = EULA_REVISION.trim() + EULADIR = "${WORKSPACE}\\giseula" + configurationhelper.gitCheckoutAndRegister(meta, 'giseula', EULAURL, EULABRANCH, EULADIR) + env.INST_EULA = EULADIR + + PREREQURL = 'git@hydra:gis/installerprerequisites.git' + PREREQBRANCH = PREREQUISITE_REVISION.trim() + PREREQDIR = "${WORKSPACE}\\prereq" + configurationhelper.gitCheckoutAndRegister(meta, 'installerprerequisites', PREREQURL, PREREQBRANCH, PREREQDIR) + env.INST_PREQ = PREREQDIR + + DIGSIGURL = 'git@hydra:gis/digitalsignature.git' + DIGSIGBRANCH = DIGITALSIGNATURE_REVISION.trim() + DIGSIGDIR = "${WORKSPACE}\\certificates" + configurationhelper.gitCheckoutAndRegister(meta, 'digitalsignature', DIGSIGURL, DIGSIGBRANCH, DIGSIGDIR) + env.PATH_TO_CERTIFICATE_PFX = "${DIGSIGDIR}\\GIS" + } + } + } + stage('Checkout source') { + steps { + script { + SOURCEURL = 'git@hydra:gis/newsw.git' + SOURCEREVISION = SOURCE_REVISION.trim() + SOURCEDIR = "${WORKSPACE}\\newsw" + configurationhelper.gitCheckoutAndRegister(meta, 'atlasservices', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + env.SOURCE_PATH = "${SOURCEDIR}" + env.INST_USB_PMB_C8_3_1030 = "${SOURCE_PATH}\\drivers\\PMB\\Gen3 USB Driver\\installed" + env.INST_CONFIGS = "${SOURCE_PATH}\\configs\\installed" + env.INST_RELEASE_NOTES = "${SOURCE_PATH}\\releasenotes" + env.INST_RESOURCEFILES = "${SOURCE_PATH}\\resourcefiles" + env.INST_ICCPROFILES = "${SOURCE_PATH}\\iccprofiles\\installed" + env.INST_SEC = "${SOURCE_PATH}\\security" + env.INST_SRC = "${SOURCE_PATH}\\src" + } + } + } + stage("Clean exports") { + steps { + script { + env.IMPORTSPATH = "${WORKSPACE}\\imports" + env.INTERMEDIATEEXPORTSPATH = "${WORKSPACE}\\info" + env.EXPORTSPATH = "installers" + env.VERSIONINFO = "${WORKSPACE}\\versioninfo" + FX2DIR = "${WORKSPACE}\\FX2USB" + FX3DIR = "${WORKSPACE}\\FX3USB" + cleanfolders = [ + IMPORTSPATH, + INTERMEDIATEEXPORTSPATH, + EXPORTSPATH, + VERSIONINFO, + FX2DIR, + FX3DIR, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Generating version info') { + when { + expression { + toBuild + } + } + steps { + script { + version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + version.generate(meta, SOURCE_REVISION, SOURCE_PATH) + } + } + } + stage("Import binaries") { + when { + expression { + toBuild + } + } + steps { + script { + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + def fx2options = importer.getNewImportOptions() + fx2options.source(PMB_FX2_ARTIFACT_LOCATION, PMB_FX2_ARTIFACT_BUILD_JOB, PMB_FX2_ARTIFACT_BUILD_JOB_SELECTOR, PMB_FX2_RELEASE_REPO, PMB_FX2_RELEASE_VERSION) + fx2options.setincrementversionnumber(false) + fx2options.setnested() + importer.artifactimport(meta, fx2options, FX2DIR) + env.INST_USB_PMB_C8_C2 = "${FX2DIR}\\windows_drivers\\pmbfx2windrivers" + + def fx3options = importer.getNewImportOptions() + fx3options.source(PMB_FX3_ARTIFACT_LOCATION, PMB_FX3_ARTIFACT_BUILD_JOB, PMB_FX3_ARTIFACT_BUILD_JOB_SELECTOR, PMB_FX3_RELEASE_REPO, PMB_FX3_RELEASE_VERSION) + fx3options.setincrementversionnumber(false) + fx3options.setnested() + importer.artifactimport(meta, fx3options, FX3DIR) + env.INST_USB_PMB_C8_3_1040_1041 = "${FX3DIR}\\windows_drivers\\pmbfx3windrivers\\GIS_Active\\bin" + + def BUILD_STAMP=meta.buildstampinfo.buildstamp + artifactDetails = [ + // Key, Artifacts Source project Artifacts source selector, Destination folder Increment Version Number Artifacts to copy Artifacts to not copy Binaries Folders + CoreSet: [PRINT_SERVER_COMPILE_JOB, PRINT_SERVER_COMPILE_BUILD_SELECTOR, "${IMPORTSPATH}\\build\\coreset", false, '**', '**/*.pdb,**/*.lib', [ 'bin', 'scripts'], ], + USBSet: [PRINT_SERVER_COMPILE_JOB, PRINT_SERVER_COMPILE_BUILD_SELECTOR, "${IMPORTSPATH}\\build\\usbset", false, '**', '**/*.lib', [ 'bin', 'scripts'], ], + LayoutSet: [OLD_LAYOUT_COMPILE_JOB, OLD_LAYOUT_COMPILE_BUILD_SELECTOR, "${IMPORTSPATH}\\build\\layoutset", false, '**', '**/*.lib', [ 'bin' ], ], + ] + installerDetails = [ + // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename + CORE_XR1003: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], + CORE_FDA4D: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'FDA4D', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/FDA4D/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/FDA4D", "GIS Inkjet OS FDA4D ${BUILD_STAMP}.exe" ], + CORE_FD1200: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'FD1200', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/FD1200/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/FD1200", "GIS Inkjet OS FD1200 ${BUILD_STAMP}.exe" ], + CORE_RH: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'RH', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/RH/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/RH", "GIS Inkjet OS RH ${BUILD_STAMP}.exe" ], + CORE_RH1280: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'RH1280', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/RH1280/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/RH1280", "GIS Inkjet OS RH1280 ${BUILD_STAMP}.exe" ], + CORE_KM: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KM', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KM/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KM", "GIS Inkjet OS KM ${BUILD_STAMP}.exe" ], + CORE_KM1800i: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KM1800i', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KM1800i/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KM1800i", "GIS Inkjet OS KM1800i ${BUILD_STAMP}.exe" ], + CORE_KM1280i: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KM1280i', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KM1280i/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KM1280i", "GIS Inkjet OS KM1280i ${BUILD_STAMP}.exe" ], + CORE_KY: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KY', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KY/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KY", "GIS Inkjet OS KY ${BUILD_STAMP}.exe" ], + CORE_TT: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'TT', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/TT/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/TT", "GIS Inkjet OS TT ${BUILD_STAMP}.exe" ], + CORE_ALL_HIBS: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'ALL', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/ALL", "GIS Inkjet OS ALL ${BUILD_STAMP}.exe" ], + USB: [INSTALLSHIELD2020_CMD, 'USBSet', 'install_usb', "${SOURCE_PATH}\\installer\\build\\prereq_usb", 'Prereq USB.ism', 'Prereq', "${INTERMEDIATEEXPORTSPATH}/install_usb/x64/Prereq/DiskImages/DISK1/GIS USB 2.exe", "${EXPORTSPATH}/USB", "GIS USB ${BUILD_STAMP}.exe" ], + LAYOUT: [INSTALLSHIELD2020_CMD, 'LayoutSet', 'install_layout', "${SOURCE_PATH}\\installer\\build\\layout", 'Layout.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_layout/x64/GIS Layout 2/DiskImages/DISK1/GIS Layout 2.exe", "${EXPORTSPATH}/Layout", "GIS Layout ${BUILD_STAMP}.exe" ], + PDF: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_pdf', "${SOURCE_PATH}\\installer\\build\\pdf", 'PDF_RIP_x64.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_pdf/x64/PDF_RIP/DiskImages/DISK1/setup_pdf 2.exe", "${EXPORTSPATH}/PDF", "GIS Inkjet OS PDF ${BUILD_STAMP}.exe" ], + TCPIP_SDK: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_sdk', "${SOURCE_PATH}\\installer\\build\\source", 'TCPIP SDK.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_sdk/x64/TCPIP/DiskImages/DISK1/GIS Inkjet 2 TcpIp SDK.exe", "${EXPORTSPATH}/TCPIPSDK", "GIS TcpIp SDK ${BUILD_STAMP}.exe" ], + TRANSPORT_SDK: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_sdk', "${SOURCE_PATH}\\installer\\build\\source", 'Transport SDK.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_sdk/x64/TRANSPORT/DiskImages/DISK1/GIS Inkjet 2 Transport SDK.exe", "${EXPORTSPATH}/TransportSDK", "GIS Transport SDK ${BUILD_STAMP}.exe" ], + ] + installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") + artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) + } + } + } + stage("Sign drivers") { + when { + expression { + toBuild + } + } + steps { + script { + for (installer in toBuild) { + if (installer == 'USB' ) { + def INF2CAT = "${WINDOWS_DRIVER_KIT_10}\\bin\\x86\\inf2cat" + def SIGNTOOL = "${WINDOWS_DRIVER_KIT_10}\\bin\\10.0.18362.0\\x86\\signtool" + def HASHALGORITHM = 'sha256' + //def TIMESTAMPSERVER = 'http://timestamp.globalsign.com/scripts/timestamp.dll' + def COMPANY_NAME = 'Global Inkjet Systems Ltd' + + INST_USB_PMB_C8_3_1030 = "${SOURCE_PATH}\\drivers\\PMB\\Gen3 USB Driver\\installed" + dir("${INST_USB_PMB_C8_3_1030}\\win7\\x64") { + retry(3) { // Just in case the time stamp server is temporarily unavailable + bat """ + "${INF2CAT}" /verbose /driver:. /os:7_X64 + "${SIGNTOOL}" sign /v /fd ${HASHALGORITHM} /n "${COMPANY_NAME}" /tr ${TIME_STAMP_SERVER.trim()} /td ${HASHALGORITHM} *.cat + """ + } + } + } + } + } + } + } + stage('Generate version info folder for installer') { + steps { + script { + def metamap = [ + [ VERSIONINFO, meta ], // Incomplete until this job ends + ] + for (metaitem in metamap) { + (destfolder, componentmeta) = metaitem + dir(destfolder) { + componentmeta.saveJsonSerialised("meta.json") + } + } + } + } + } + stage("Build installers") { + when { + expression { + toBuild + } + } + steps { + script { + fileOperations([folderCreateOperation(INTERMEDIATEEXPORTSPATH)]) + installerpublishlist = [] + for (installer in toBuild) { + (installcmd, artifactKey, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(installer) + + def importmeta = artifactFetched.get(artifactKey) + + echo "For: ${installer} build stamp is: ${importmeta.buildstampinfo.buildstamp}" + def INST_ISVERSION = importmeta.buildstampinfo.buildstamp + + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + def INST_BIN = "${destFolder}\\bin" + + def INST_VERBOSE = '' + if (VERBOSE == "true") { + INST_VERBOSE = '-v' + } + + def INST_OUT = "${INTERMEDIATEEXPORTSPATH}\\${processingFolder}" + fileOperations([folderCreateOperation(INST_OUT)]) + + def INST_VARIANT = '' + if (VARIANT != "" ) { + INST_VARIANT = '-r ' + VARIANT + } + installerhelper.checkISMfile("${EXECUTIONPATH}\\${ISMPROJECT}") + installerhelper.selectSigningCertificate(NODE_NAME, "${EXECUTIONPATH}\\${ISMPROJECT}") + + dir(EXECUTIONPATH) { + // INSTALLSHIELD201X_CMD and DOTNET_FRAMEWORK64_PATH must be externally defined + bat """ + "${installcmd}" -t "${DOTNET_FRAMEWORK64_PATH}" -p "${ISMPROJECT}" -y "${INST_ISVERSION}" -l "WORKSPACE=${WORKSPACE}" -l "INST_USB_PMB_C8_3_1040_1041=${INST_USB_PMB_C8_3_1040_1041}" -l "INST_USB_PMB_C8_3_1030=${INST_USB_PMB_C8_3_1030}" -l "INST_USB_PMB_C8_C2=${INST_USB_PMB_C8_C2}" -l "INST_BIN=${INST_BIN}" -l "INST_CONFIGS=${INST_CONFIGS}" -l "INST_RELEASE_NOTES=${INST_RELEASE_NOTES}" -l "INST_RESOURCEFILES=${INST_RESOURCEFILES}" -l "INST_SRC=${INST_SRC}" -l "INST_OUT=${INST_OUT}" -l "INST_EULA=${INST_EULA}" -l "INST_ICCPROFILES=${INST_ICCPROFILES}" -l "INST_SUPPORT=${INST_SUPPORT}" -l "INST_SEC=${INST_SEC}" -l "INST_VERSIONINFO=${VERSIONINFO}" -prqpath "${INST_PREQ}" ${INST_VERBOSE} ${INST_VARIANT} -a x64 + """ + } + + installerhelper.copyToFinalPosition(pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) + + def EXPORT_OUT_GLOB = "${FINALBASEFOLDER}/**" + echo "Adding ${EXPORT_OUT_GLOB} to installer publish list" + installerpublishlist.add(EXPORT_OUT_GLOB) + } + } + } + } + stage('Classifying publishable artifacts') { + when { + expression { + toBuild + } + } + steps { + script { + def publishable = installerpublishlist + publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") + publishutils.generatelist(meta, publishable) + } + } + } + stage('Archiving artifacts') { + when { + expression { + toBuild + } + } + steps { + script { + ArrayList itemsForRelease = ["info/**", "installers/**"] + def artifactlist = meta.generateartifacts(itemsForRelease) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + stage('Check for unstable build') { + when { + expression { + toBuild + } + } + steps { + script { + def installersToBuild = installerDetails.size() + if (installersToBuild != toBuild.size()) { + echo "Not building the complete set of installers (${installersToBuild}) therefore marking building unstable" + currentBuild.result = 'UNSTABLE' + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} From 1b96d3cc7e481cd86dcb6aa1acacb475da936a89 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 11:52:49 +0000 Subject: [PATCH 16/73] CORE-594 - Use an initial ISM file --- install/install.groovy | 132 +++++++++++++++++------------------------ 1 file changed, 53 insertions(+), 79 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 346d06c..3a7b1ec 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -1,3 +1,34 @@ +def lnpubuildinfo = null + +/* Example expected JSON +{ + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "LNPU_master/LNPU_master_build", + "Job selector": "true", + "Release repo": "git@hydra:local-network-package-updater-releases/local-network-package-updater-whole.git", + "Release repo revision": "" + } +} +*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: lnpubuildinfo, pretty: 1) +} + +def boolean exists() { + return (lnpubuildinfo != null) +} + +def set(String jsontext) { + lnpubuildinfo = readJSON(text: jsontext) + echo "LNPU build info:\n${asJsonText()}" +} + +def get() { + return lnpubuildinfo +} + pipeline { agent { label AGENT_LABEL } stages { @@ -6,6 +37,9 @@ pipeline { script { meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + set(LNPU_BINARIES_INFO) + lnpubuildjobartifactinfo = get()['Artifact info'] + if (INSTALLSHIELD2020_CMD == "" || DOTNET_FRAMEWORK64_PATH == "" || WINDOWS_DRIVER_KIT_8_1 == "" || WINDOWS_DRIVER_KIT_10 == "") { echo "Either INSTALLSHIELD2020_CMD, DOTNET_FRAMEWORK64_PATH, WINDOWS_DRIVER_KIT_8_1 or WINDOWS_DRIVER_KIT_10 environment variables have not been defined" currentBuild.result = 'FAILED' @@ -86,17 +120,11 @@ pipeline { stage('Checkout source') { steps { script { - SOURCEURL = 'git@hydra:gis/newsw.git' + SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = SOURCE_REVISION.trim() - SOURCEDIR = "${WORKSPACE}\\newsw" - configurationhelper.gitCheckoutAndRegister(meta, 'atlasservices', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - env.SOURCE_PATH = "${SOURCEDIR}" - env.INST_USB_PMB_C8_3_1030 = "${SOURCE_PATH}\\drivers\\PMB\\Gen3 USB Driver\\installed" - env.INST_CONFIGS = "${SOURCE_PATH}\\configs\\installed" - env.INST_RELEASE_NOTES = "${SOURCE_PATH}\\releasenotes" - env.INST_RESOURCEFILES = "${SOURCE_PATH}\\resourcefiles" - env.INST_ICCPROFILES = "${SOURCE_PATH}\\iccprofiles\\installed" - env.INST_SEC = "${SOURCE_PATH}\\security" + SOURCEDIR = "${WORKSPACE}\\lnpu" + configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + env.SOURCE_PATH = SOURCEDIR env.INST_SRC = "${SOURCE_PATH}\\src" } } @@ -108,15 +136,11 @@ pipeline { env.INTERMEDIATEEXPORTSPATH = "${WORKSPACE}\\info" env.EXPORTSPATH = "installers" env.VERSIONINFO = "${WORKSPACE}\\versioninfo" - FX2DIR = "${WORKSPACE}\\FX2USB" - FX3DIR = "${WORKSPACE}\\FX3USB" cleanfolders = [ IMPORTSPATH, INTERMEDIATEEXPORTSPATH, EXPORTSPATH, VERSIONINFO, - FX2DIR, - FX3DIR, ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -145,82 +169,32 @@ pipeline { steps { script { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - def fx2options = importer.getNewImportOptions() - fx2options.source(PMB_FX2_ARTIFACT_LOCATION, PMB_FX2_ARTIFACT_BUILD_JOB, PMB_FX2_ARTIFACT_BUILD_JOB_SELECTOR, PMB_FX2_RELEASE_REPO, PMB_FX2_RELEASE_VERSION) - fx2options.setincrementversionnumber(false) - fx2options.setnested() - importer.artifactimport(meta, fx2options, FX2DIR) - env.INST_USB_PMB_C8_C2 = "${FX2DIR}\\windows_drivers\\pmbfx2windrivers" - - def fx3options = importer.getNewImportOptions() - fx3options.source(PMB_FX3_ARTIFACT_LOCATION, PMB_FX3_ARTIFACT_BUILD_JOB, PMB_FX3_ARTIFACT_BUILD_JOB_SELECTOR, PMB_FX3_RELEASE_REPO, PMB_FX3_RELEASE_VERSION) - fx3options.setincrementversionnumber(false) - fx3options.setnested() - importer.artifactimport(meta, fx3options, FX3DIR) - env.INST_USB_PMB_C8_3_1040_1041 = "${FX3DIR}\\windows_drivers\\pmbfx3windrivers\\GIS_Active\\bin" + + def lnpubuildimportoptions = importer.getNewImportOptions() + lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], + lnpubuildjobartifactinfo['Job'], + lnpubuildjobartifactinfo['Job selector'], + lnpubuildjobartifactinfo['Release repo'], + lnpubuildjobartifactinfo['Release repo revision']) + lnpubuildimportoptions.setincrementversionnumber(false) + lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') + def BUILD_STAMP=meta.buildstampinfo.buildstamp artifactDetails = [ - // Key, Artifacts Source project Artifacts source selector, Destination folder Increment Version Number Artifacts to copy Artifacts to not copy Binaries Folders - CoreSet: [PRINT_SERVER_COMPILE_JOB, PRINT_SERVER_COMPILE_BUILD_SELECTOR, "${IMPORTSPATH}\\build\\coreset", false, '**', '**/*.pdb,**/*.lib', [ 'bin', 'scripts'], ], - USBSet: [PRINT_SERVER_COMPILE_JOB, PRINT_SERVER_COMPILE_BUILD_SELECTOR, "${IMPORTSPATH}\\build\\usbset", false, '**', '**/*.lib', [ 'bin', 'scripts'], ], - LayoutSet: [OLD_LAYOUT_COMPILE_JOB, OLD_LAYOUT_COMPILE_BUILD_SELECTOR, "${IMPORTSPATH}\\build\\layoutset", false, '**', '**/*.lib', [ 'bin' ], ], + // Key, Artifacts Source project Artifacts source selector, Destination folder Increment Version Number Artifacts to copy Artifacts to not copy Binaries Folders + AppPackageSet: [PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, "${IMPORTSPATH}\\build\\apppackage", false, '**', '**/*.pdb,**/*.lib', [ 'bin', 'scripts'], ], + ControlPackageSet: [CONTROL_PACKAGE_CREATE_JOB, CONTROL_PACKAGE_CREATE_JOB_SELECTOR, "${IMPORTSPATH}\\build\\controlpackage", false, '**', '**/*.lib', [ 'bin', 'scripts'], ], ] installerDetails = [ - // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename - CORE_XR1003: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], - CORE_FDA4D: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'FDA4D', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/FDA4D/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/FDA4D", "GIS Inkjet OS FDA4D ${BUILD_STAMP}.exe" ], - CORE_FD1200: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'FD1200', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/FD1200/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/FD1200", "GIS Inkjet OS FD1200 ${BUILD_STAMP}.exe" ], - CORE_RH: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'RH', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/RH/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/RH", "GIS Inkjet OS RH ${BUILD_STAMP}.exe" ], - CORE_RH1280: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'RH1280', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/RH1280/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/RH1280", "GIS Inkjet OS RH1280 ${BUILD_STAMP}.exe" ], - CORE_KM: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KM', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KM/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KM", "GIS Inkjet OS KM ${BUILD_STAMP}.exe" ], - CORE_KM1800i: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KM1800i', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KM1800i/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KM1800i", "GIS Inkjet OS KM1800i ${BUILD_STAMP}.exe" ], - CORE_KM1280i: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KM1280i', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KM1280i/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KM1280i", "GIS Inkjet OS KM1280i ${BUILD_STAMP}.exe" ], - CORE_KY: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'KY', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/KY/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/KY", "GIS Inkjet OS KY ${BUILD_STAMP}.exe" ], - CORE_TT: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'TT', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/TT/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/TT", "GIS Inkjet OS TT ${BUILD_STAMP}.exe" ], - CORE_ALL_HIBS: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'ALL', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/ALL", "GIS Inkjet OS ALL ${BUILD_STAMP}.exe" ], - USB: [INSTALLSHIELD2020_CMD, 'USBSet', 'install_usb', "${SOURCE_PATH}\\installer\\build\\prereq_usb", 'Prereq USB.ism', 'Prereq', "${INTERMEDIATEEXPORTSPATH}/install_usb/x64/Prereq/DiskImages/DISK1/GIS USB 2.exe", "${EXPORTSPATH}/USB", "GIS USB ${BUILD_STAMP}.exe" ], - LAYOUT: [INSTALLSHIELD2020_CMD, 'LayoutSet', 'install_layout', "${SOURCE_PATH}\\installer\\build\\layout", 'Layout.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_layout/x64/GIS Layout 2/DiskImages/DISK1/GIS Layout 2.exe", "${EXPORTSPATH}/Layout", "GIS Layout ${BUILD_STAMP}.exe" ], - PDF: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_pdf', "${SOURCE_PATH}\\installer\\build\\pdf", 'PDF_RIP_x64.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_pdf/x64/PDF_RIP/DiskImages/DISK1/setup_pdf 2.exe", "${EXPORTSPATH}/PDF", "GIS Inkjet OS PDF ${BUILD_STAMP}.exe" ], - TCPIP_SDK: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_sdk', "${SOURCE_PATH}\\installer\\build\\source", 'TCPIP SDK.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_sdk/x64/TCPIP/DiskImages/DISK1/GIS Inkjet 2 TcpIp SDK.exe", "${EXPORTSPATH}/TCPIPSDK", "GIS TcpIp SDK ${BUILD_STAMP}.exe" ], - TRANSPORT_SDK: [INSTALLSHIELD2020_CMD, 'CoreSet', 'install_sdk', "${SOURCE_PATH}\\installer\\build\\source", 'Transport SDK.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_sdk/x64/TRANSPORT/DiskImages/DISK1/GIS Inkjet 2 Transport SDK.exe", "${EXPORTSPATH}/TransportSDK", "GIS Transport SDK ${BUILD_STAMP}.exe" ], + // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename + LNPU: [INSTALLSHIELD2020_CMD, 'AppPackageSet,ControlPackageSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) } } } - stage("Sign drivers") { - when { - expression { - toBuild - } - } - steps { - script { - for (installer in toBuild) { - if (installer == 'USB' ) { - def INF2CAT = "${WINDOWS_DRIVER_KIT_10}\\bin\\x86\\inf2cat" - def SIGNTOOL = "${WINDOWS_DRIVER_KIT_10}\\bin\\10.0.18362.0\\x86\\signtool" - def HASHALGORITHM = 'sha256' - //def TIMESTAMPSERVER = 'http://timestamp.globalsign.com/scripts/timestamp.dll' - def COMPANY_NAME = 'Global Inkjet Systems Ltd' - - INST_USB_PMB_C8_3_1030 = "${SOURCE_PATH}\\drivers\\PMB\\Gen3 USB Driver\\installed" - dir("${INST_USB_PMB_C8_3_1030}\\win7\\x64") { - retry(3) { // Just in case the time stamp server is temporarily unavailable - bat """ - "${INF2CAT}" /verbose /driver:. /os:7_X64 - "${SIGNTOOL}" sign /v /fd ${HASHALGORITHM} /n "${COMPANY_NAME}" /tr ${TIME_STAMP_SERVER.trim()} /td ${HASHALGORITHM} *.cat - """ - } - } - } - } - } - } - } stage('Generate version info folder for installer') { steps { script { From 99024d4c3fbd4cf43508d401c4d74ec3c92a17a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 12:05:51 +0000 Subject: [PATCH 17/73] CORE-594 - Debug installer creation --- install/install.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 3a7b1ec..d852818 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -134,13 +134,16 @@ pipeline { script { env.IMPORTSPATH = "${WORKSPACE}\\imports" env.INTERMEDIATEEXPORTSPATH = "${WORKSPACE}\\info" - env.EXPORTSPATH = "installers" + env.EXPORTSPATH = 'installers' env.VERSIONINFO = "${WORKSPACE}\\versioninfo" + LNPUBUILDIMPORT = 'lnpubuild' + env.LNPUBUILD = "${WORKSPACE}\\${LNPUBUILDIMPORT}" cleanfolders = [ IMPORTSPATH, INTERMEDIATEEXPORTSPATH, EXPORTSPATH, VERSIONINFO, + LNPUBUILD, ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -178,7 +181,7 @@ pipeline { lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') - + lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) def BUILD_STAMP=meta.buildstampinfo.buildstamp artifactDetails = [ From 6879adacb2f685ea7be4fea3cbffbbd5c827b40a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 12:13:57 +0000 Subject: [PATCH 18/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index d852818..c8882b3 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -176,7 +176,7 @@ pipeline { def lnpubuildimportoptions = importer.getNewImportOptions() lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], lnpubuildjobartifactinfo['Job'], - lnpubuildjobartifactinfo['Job selector'], + "${lnpubuildjobartifactinfo['Job selector']}", lnpubuildjobartifactinfo['Release repo'], lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) From c7cb16419086d915f1420f73efad75238af64943 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 12:38:49 +0000 Subject: [PATCH 19/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 5 +++++ install/install.groovy | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index bb0d32e..8df88e1 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -45,6 +45,10 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } + void show() { + script.echo(this.toString()) + } + void setsourceselect(String sourceselect) { sourceselect = sourceselect.trim() switch (sourceselect) { @@ -263,6 +267,7 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { } def artifactimport(meta, ImportOptions importoptions, String destpath) { + importoptions.show() switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': diff --git a/install/install.groovy b/install/install.groovy index c8882b3..df1afcc 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -143,7 +143,7 @@ pipeline { INTERMEDIATEEXPORTSPATH, EXPORTSPATH, VERSIONINFO, - LNPUBUILD, + LNPUBUILDIMPORT, ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) From 6f34d797d8797f07ada650a0657d4ec264d00bc9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 12:40:52 +0000 Subject: [PATCH 20/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index 8df88e1..8bc96a3 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -46,7 +46,7 @@ class ImportOptions { } void show() { - script.echo(this.toString()) + m_script.echo(this.toString()) } void setsourceselect(String sourceselect) { From 50d495695c9301c7050c5d9a8f277a1b7be406aa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 12:49:08 +0000 Subject: [PATCH 21/73] CORE-594 - Debug installer creation --- install/install.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/install.groovy b/install/install.groovy index df1afcc..1b59928 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -182,6 +182,8 @@ pipeline { lnpubuildimportoptions.setincrementversionnumber(false) lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) + + echo ("I am here") def BUILD_STAMP=meta.buildstampinfo.buildstamp artifactDetails = [ @@ -193,6 +195,7 @@ pipeline { // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename LNPU: [INSTALLSHIELD2020_CMD, 'AppPackageSet,ControlPackageSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] + echo ("artifactDetails: ${artifactDetails}\ninstallerDetails: ${installerDetails}") installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) } From 6941daeb299d34e05ced43a77f7bc679b648f297 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 12:52:47 +0000 Subject: [PATCH 22/73] CORE-594 - Debug installer creation --- install/install.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 1b59928..b3d41e2 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -173,6 +173,8 @@ pipeline { script { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + echo ("I am here: ${lnpubuildjobartifactinfo['Location']} ${lnpubuildjobartifactinfo['Job']} ${lnpubuildjobartifactinfo['Job selector']}") + def lnpubuildimportoptions = importer.getNewImportOptions() lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], lnpubuildjobartifactinfo['Job'], @@ -183,7 +185,7 @@ pipeline { lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) - echo ("I am here") + echo ("I am here 2") def BUILD_STAMP=meta.buildstampinfo.buildstamp artifactDetails = [ From 7bb00e3045152506e9be5cc4f151b195239b0658 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:03:32 +0000 Subject: [PATCH 23/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 20 +++++++++++++++++--- install/install.groovy | 1 + 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index 8bc96a3..2f7611e 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -45,8 +45,22 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void show() { - m_script.echo(this.toString()) + String show() { + return """m_incrementversionnumber: ${m_incrementversionnumber} + m_filter: ${m_filter} + m_excludes: ${m_excludes} + m_fingerprint: ${m_fingerprint} + m_nested: ${m_nested} + m_flatten: ${m_flatten} + m_includemetainfilter: ${m_includemetainfilter} + m_clean: ${m_clean} + m_appendtoparentmeta: ${m_appendtoparentmeta} + m_sourceselect: ${m_sourceselect} + m_jobname: ${m_jobname} + m_buildnumberorselector: ${m_buildnumberorselector} + m_repo: ${m_repo} + m_revision: ${m_revision} + """ } void setsourceselect(String sourceselect) { @@ -267,7 +281,7 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { } def artifactimport(meta, ImportOptions importoptions, String destpath) { - importoptions.show() + echo "Importing artifacts: Import options:\n${importoptions.show()}\n to: ${destpath}" switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': diff --git a/install/install.groovy b/install/install.groovy index b3d41e2..1fc6ace 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -174,6 +174,7 @@ pipeline { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") echo ("I am here: ${lnpubuildjobartifactinfo['Location']} ${lnpubuildjobartifactinfo['Job']} ${lnpubuildjobartifactinfo['Job selector']}") + echo ("I am here: ${lnpubuildjobartifactinfo['Release repo']} ${lnpubuildjobartifactinfo['Release repo revision']}") def lnpubuildimportoptions = importer.getNewImportOptions() lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], From 3038e66204d7e62a0f9df4b49af89e92e018782a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:21:01 +0000 Subject: [PATCH 24/73] CORE-594 - Debug installer creation --- install/install.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/install.groovy b/install/install.groovy index 1fc6ace..3f4b4de 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -177,6 +177,7 @@ pipeline { echo ("I am here: ${lnpubuildjobartifactinfo['Release repo']} ${lnpubuildjobartifactinfo['Release repo revision']}") def lnpubuildimportoptions = importer.getNewImportOptions() +/* lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], lnpubuildjobartifactinfo['Job'], "${lnpubuildjobartifactinfo['Job selector']}", @@ -184,6 +185,8 @@ pipeline { lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') + */ + lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) echo ("I am here 2") From 6f192be0d3473585debe8df53cb638d391d0376d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:22:27 +0000 Subject: [PATCH 25/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 3f4b4de..7c2a431 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -184,8 +184,8 @@ pipeline { lnpubuildjobartifactinfo['Release repo'], lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) - lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') */ + //lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) From 217b7a1def7196bad1b8a9daeb29d0b21f860464 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:29:58 +0000 Subject: [PATCH 26/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 7c2a431..83553f9 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -186,7 +186,7 @@ pipeline { lnpubuildimportoptions.setincrementversionnumber(false) */ //lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') - lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") + lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", 8) lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) echo ("I am here 2") From b77883f1abe1b0bc2d29a6a6f7b9b1084b187659 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:35:50 +0000 Subject: [PATCH 27/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index 2f7611e..af40c5f 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -127,7 +127,7 @@ class ImportOptions { m_jobname = jobname.trim() m_buildnumberorselector = [ $class: 'ParameterizedBuildSelector', - parameterName: "${buildnumberorselector}" + parameterName: "${buildnumberorselector}" ] setsourceselect('Jenkins job artifact') } From 70e4f46c1e107f057b76b9587fc310dd6971694b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:39:13 +0000 Subject: [PATCH 28/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 2 +- install/install.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index af40c5f..b2dc41b 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -114,7 +114,7 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void jenkinsjob(String jobname, buildnumberorselector) { + void jenkinsjob(String jobname, String buildnumberorselector) { m_jobname = jobname.trim() m_buildnumberorselector = [ $class: 'ParameterizedBuildSelector', diff --git a/install/install.groovy b/install/install.groovy index 83553f9..5445dd1 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -186,7 +186,7 @@ pipeline { lnpubuildimportoptions.setincrementversionnumber(false) */ //lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') - lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", 8) + lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "8") lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) echo ("I am here 2") From a7ef1d3fd4ef247bece6d60d89be3387a9b9725e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:41:49 +0000 Subject: [PATCH 29/73] CORE-594 - Debug installer creation --- install/install.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 5445dd1..5992a39 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -186,7 +186,8 @@ pipeline { lnpubuildimportoptions.setincrementversionnumber(false) */ //lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') - lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "8") + // Works lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "8") + lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) echo ("I am here 2") From 2ceddfde8f34d0788bc36bcbb1bbc681c5c5df88 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 13:47:16 +0000 Subject: [PATCH 30/73] CORE-594 - Debug installer creation --- install/install.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 5992a39..c7bae90 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -187,7 +187,9 @@ pipeline { */ //lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') // Works lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "8") - lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") + // Not working lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") + lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", EXP_LNPU_SELECTOR) + lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) echo ("I am here 2") From 4e0c7b19f60519c92c8683d3be51cd2f128b8a38 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:06:47 +0000 Subject: [PATCH 31/73] CORE-594 - Debug installer creation --- install/install.groovy | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index c7bae90..0c7eb92 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -5,7 +5,7 @@ def lnpubuildinfo = null "Artifact info": { "Location": "Jenkins job artifact", "Job": "LNPU_master/LNPU_master_build", - "Job selector": "true", + "Job selector": "true", "Release repo": "git@hydra:local-network-package-updater-releases/local-network-package-updater-whole.git", "Release repo revision": "" } @@ -177,21 +177,13 @@ pipeline { echo ("I am here: ${lnpubuildjobartifactinfo['Release repo']} ${lnpubuildjobartifactinfo['Release repo revision']}") def lnpubuildimportoptions = importer.getNewImportOptions() -/* lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], lnpubuildjobartifactinfo['Job'], - "${lnpubuildjobartifactinfo['Job selector']}", + lnpubuildjobartifactinfo['Job selector'], lnpubuildjobartifactinfo['Release repo'], lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) - */ - //lnpubuildimportoptions.setexcludes('**/*.pdb,**/*.lib') - // Works lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "8") - // Not working lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", "true") - lnpubuildimportoptions.jenkinsjob("LNPU_master/LNPU_master_build", EXP_LNPU_SELECTOR) - lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) - echo ("I am here 2") def BUILD_STAMP=meta.buildstampinfo.buildstamp From fda3e8890a695b52c301d75037a1ef303f807abf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:16:27 +0000 Subject: [PATCH 32/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 10 ++++++++-- Common_scripts/install/installhelper.groovy | 1 + install/install.groovy | 5 ----- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index b2dc41b..d6fb4d4 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -20,6 +20,8 @@ class ImportOptions { String m_repo String m_revision + + boolean m_showdetails ImportOptions(script, String sourceselect = 'Jenkins job artifact', @@ -31,7 +33,8 @@ class ImportOptions { boolean flatten = false, boolean includemetainfilter = true, boolean clean = true, - appendtoparentmeta = true) { + appendtoparentmeta = true, + show = false) { m_script = script m_sourceselect = sourceselect.trim() m_incrementversionnumber = incrementversionnumber @@ -43,6 +46,7 @@ class ImportOptions { m_includemetainfilter = includemetainfilter m_clean = clean m_appendtoparentmeta = appendtoparentmeta + m_showdetails = show } String show() { @@ -281,7 +285,9 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { } def artifactimport(meta, ImportOptions importoptions, String destpath) { - echo "Importing artifacts: Import options:\n${importoptions.show()}\n to: ${destpath}" + if (importoptions.m_show) { + echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" + } switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 96a8b2e..2915d0c 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -26,6 +26,7 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { artifactFetched = [:] + echo ("artifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") def importoptions = importer.getNewImportOptions() for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] diff --git a/install/install.groovy b/install/install.groovy index 0c7eb92..b68ca31 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -173,9 +173,6 @@ pipeline { script { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - echo ("I am here: ${lnpubuildjobartifactinfo['Location']} ${lnpubuildjobartifactinfo['Job']} ${lnpubuildjobartifactinfo['Job selector']}") - echo ("I am here: ${lnpubuildjobartifactinfo['Release repo']} ${lnpubuildjobartifactinfo['Release repo revision']}") - def lnpubuildimportoptions = importer.getNewImportOptions() lnpubuildimportoptions.source(lnpubuildjobartifactinfo['Location'], lnpubuildjobartifactinfo['Job'], @@ -184,7 +181,6 @@ pipeline { lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) - echo ("I am here 2") def BUILD_STAMP=meta.buildstampinfo.buildstamp artifactDetails = [ @@ -196,7 +192,6 @@ pipeline { // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename LNPU: [INSTALLSHIELD2020_CMD, 'AppPackageSet,ControlPackageSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] - echo ("artifactDetails: ${artifactDetails}\ninstallerDetails: ${installerDetails}") installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) } From 8ec1a7726bbc3ad2a19d79029c83dd96a0eadc1a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:21:53 +0000 Subject: [PATCH 33/73] CORE-594 - Debug installer creation --- Common_scripts/import/importer.groovy | 10 +++++++--- Common_scripts/install/installhelper.groovy | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy index d6fb4d4..4948cad 100644 --- a/Common_scripts/import/importer.groovy +++ b/Common_scripts/import/importer.groovy @@ -34,7 +34,7 @@ class ImportOptions { boolean includemetainfilter = true, boolean clean = true, appendtoparentmeta = true, - show = false) { + showdetails = false) { m_script = script m_sourceselect = sourceselect.trim() m_incrementversionnumber = incrementversionnumber @@ -46,7 +46,7 @@ class ImportOptions { m_includemetainfilter = includemetainfilter m_clean = clean m_appendtoparentmeta = appendtoparentmeta - m_showdetails = show + m_showdetails = showdetails } String show() { @@ -147,6 +147,10 @@ class ImportOptions { releaserepo(repo, revision) setsourceselect(sourceselect) } + + void showdetails(boolean showdetails) { + m_showdetails = showdetails + } } ImportOptions getNewImportOptions() { @@ -285,7 +289,7 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { } def artifactimport(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_show) { + if (importoptions.m_showdetails) { echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" } switch (importoptions.m_sourceselect) { diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 2915d0c..9c11499 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -23,11 +23,12 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boolean showdetails = false) { artifactFetched = [:] echo ("artifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") def importoptions = importer.getNewImportOptions() + importoptions.showdetails(showdetails) for (installer in toBuild) { def artifactKeys = installerDetails.get(installer)[1] if (artifactKeys instanceof java.lang.String) { From d121af7eadd189d7d429c0526f638ffb97c49223 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:29:40 +0000 Subject: [PATCH 34/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 4 +++- install/install.groovy | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 9c11499..214fa84 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -26,7 +26,9 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boolean showdetails = false) { artifactFetched = [:] - echo ("artifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") + if (showdetails) { + echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") + } def importoptions = importer.getNewImportOptions() importoptions.showdetails(showdetails) for (installer in toBuild) { diff --git a/install/install.groovy b/install/install.groovy index b68ca31..41a8d5b 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -193,7 +193,7 @@ pipeline { LNPU: [INSTALLSHIELD2020_CMD, 'AppPackageSet,ControlPackageSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") - artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) + artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails, true) } } } From 615c7d01b2e64915d2dc7d93848b0133437e6545 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:36:22 +0000 Subject: [PATCH 35/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 214fa84..a637fac 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -32,10 +32,17 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boo def importoptions = importer.getNewImportOptions() importoptions.showdetails(showdetails) for (installer in toBuild) { - def artifactKeys = installerDetails.get(installer)[1] + def installerinfo = installerDetails.get(installer) + if (showdetails) { + echo ("installer:\n${installer}\installerinfo:\n${installerinfo}") + } + def artifactKeys = installerinfo[1] if (artifactKeys instanceof java.lang.String) { artifactKeys = [ artifactKeys ] } + if (showdetails) { + echo ("artifactKeys:\n${artifactKeys}") + } for (artifactKey in artifactKeys) { (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { From 504a41ef2de7408337b051483e0012aaa7e7e57f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:38:48 +0000 Subject: [PATCH 36/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index a637fac..926775e 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -34,7 +34,7 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boo for (installer in toBuild) { def installerinfo = installerDetails.get(installer) if (showdetails) { - echo ("installer:\n${installer}\installerinfo:\n${installerinfo}") + echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") } def artifactKeys = installerinfo[1] if (artifactKeys instanceof java.lang.String) { From a27831342e6af0a89bd2dddc64bbd3c91f087153 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:44:49 +0000 Subject: [PATCH 37/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 926775e..3a1cd23 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -44,7 +44,11 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boo echo ("artifactKeys:\n${artifactKeys}") } for (artifactKey in artifactKeys) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + artifactDetail = artifactDetails.get(artifactKey) + if (showdetails) { + echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") + } + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail if (artifactFetched.get(artifactKey) == null) { importoptions.jenkinsjob(sourceProject, sourceselector) importoptions.setincrementversionnumber(incrementversionnumber) From 1a52257992ea15483655aa5eaa804186a893f1e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 14:47:26 +0000 Subject: [PATCH 38/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 41a8d5b..68c6288 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -190,7 +190,7 @@ pipeline { ] installerDetails = [ // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename - LNPU: [INSTALLSHIELD2020_CMD, 'AppPackageSet,ControlPackageSet', 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], + LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails, true) From 4b5f5db21ba2a3a095e3ceacdfc3fd4cdf4252f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 15:01:38 +0000 Subject: [PATCH 39/73] CORE-594 - Debug installer creation --- install/install.groovy | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 68c6288..e869c0d 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -193,7 +193,7 @@ pipeline { LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") - artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails, true) + artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) } } } @@ -226,9 +226,8 @@ pipeline { (installcmd, artifactKey, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(installer) def importmeta = artifactFetched.get(artifactKey) - - echo "For: ${installer} build stamp is: ${importmeta.buildstampinfo.buildstamp}" - def INST_ISVERSION = importmeta.buildstampinfo.buildstamp + def INST_ISVERSION = importmeta.buildstampinfo.buildstamp[0] + echo "For: ${installer} build stamp is: ${INST_ISVERSION}" (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) def INST_BIN = "${destFolder}\\bin" From 80a43cfc6520b90ae5e4720a42badbb7d9ebc8ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 15:03:36 +0000 Subject: [PATCH 40/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index e869c0d..94b7e70 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -226,7 +226,7 @@ pipeline { (installcmd, artifactKey, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(installer) def importmeta = artifactFetched.get(artifactKey) - def INST_ISVERSION = importmeta.buildstampinfo.buildstamp[0] + def INST_ISVERSION = importmeta[0].buildstampinfo.buildstamp echo "For: ${installer} build stamp is: ${INST_ISVERSION}" (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) From 7f19a149a896216d7fd31b8e58dc00b06be33b8e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 15:07:57 +0000 Subject: [PATCH 41/73] CORE-594 - Debug installer creation --- install/install.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 94b7e70..bbfee2e 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -226,7 +226,8 @@ pipeline { (installcmd, artifactKey, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(installer) def importmeta = artifactFetched.get(artifactKey) - def INST_ISVERSION = importmeta[0].buildstampinfo.buildstamp + echo "importmeta: ${importmeta}" + def INST_ISVERSION = importmeta['AppPackageSet'].buildstampinfo.buildstamp echo "For: ${installer} build stamp is: ${INST_ISVERSION}" (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) From 5a139618c0b8839f4d8e520b6b2bbcd9ec45cf0c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 15:12:47 +0000 Subject: [PATCH 42/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 3 +++ install/install.groovy | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 3a1cd23..77b917c 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -60,6 +60,9 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boo } } } + if (showdetails) { + echo ("artifactFetched:\n${artifactFetched}") + } return artifactFetched } diff --git a/install/install.groovy b/install/install.groovy index bbfee2e..2a0b75b 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -193,7 +193,7 @@ pipeline { LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") - artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails) + artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails, true) } } } From 740b08cf4854f463ea6b7939ae7d71c8312211ec Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 15:15:59 +0000 Subject: [PATCH 43/73] CORE-594 - Debug installer creation --- install/install.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 2a0b75b..f72c9b2 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -225,9 +225,8 @@ pipeline { for (installer in toBuild) { (installcmd, artifactKey, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(installer) - def importmeta = artifactFetched.get(artifactKey) - echo "importmeta: ${importmeta}" - def INST_ISVERSION = importmeta['AppPackageSet'].buildstampinfo.buildstamp + def importmeta = artifactFetched.get('AppPackageSet') + def INST_ISVERSION = importmeta.buildstampinfo.buildstamp echo "For: ${installer} build stamp is: ${INST_ISVERSION}" (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) From 4f193d3ab062a31de0af4a7bef338be5847f0f97 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 16:17:51 +0000 Subject: [PATCH 44/73] CORE-594 - Debug installer creation --- install/install.groovy | 149 +++++++++++++++-------------------------- 1 file changed, 54 insertions(+), 95 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index f72c9b2..1568e93 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -45,42 +45,12 @@ pipeline { currentBuild.result = 'FAILED' exit(1) } - installerlist = env.INSTALLERS_TO_BUILD - if (installerlist != null) { - installerlist = installerlist.trim() - } else { - installerlist = '' - } - toBuild = [] - if (installerlist) { - toBuildUntrimmed = installerlist.split(',') - toBuildUntrimmed.each { - toBuild.add(it.trim()) - } - } - } - echo "Installer list: ${installerlist}" - } - } - stage('Nothing to build check') { - when { - expression { - !toBuild - } - } - steps { - echo 'No installers to build' - script { - currentBuild.result = 'UNSTABLE' + + toBuild = ['LNPU'] } } } stage('Checkout dependencies') { - when { - expression { - toBuild - } - } steps { script { configurationhelper = autobuilder.configurationhelper() @@ -151,11 +121,6 @@ pipeline { } } stage('Generating version info') { - when { - expression { - toBuild - } - } steps { script { version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") @@ -164,11 +129,6 @@ pipeline { } } stage("Import binaries") { - when { - expression { - toBuild - } - } steps { script { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") @@ -190,7 +150,7 @@ pipeline { ] installerDetails = [ // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename - LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', 'XR1003', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], + LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails, true) @@ -213,62 +173,71 @@ pipeline { } } stage("Build installers") { - when { - expression { - toBuild - } - } steps { script { fileOperations([folderCreateOperation(INTERMEDIATEEXPORTSPATH)]) installerpublishlist = [] - for (installer in toBuild) { - (installcmd, artifactKey, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(installer) + (installcmd, artifactKeySet, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(toBuild) - def importmeta = artifactFetched.get('AppPackageSet') - def INST_ISVERSION = importmeta.buildstampinfo.buildstamp - echo "For: ${installer} build stamp is: ${INST_ISVERSION}" + def primaryArtifactKey = 'AppPackageSet' + def appimportmeta = artifactFetched.get(primaryArtifactKey) + def INST_ISVERSION = appimportmeta.buildstampinfo.buildstamp + echo "For: ${installer} build stamp is: ${INST_ISVERSION}" - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) - def INST_BIN = "${destFolder}\\bin" + (apppackagesourceProject, + apppackagesourceselector, + apppackagedestFolder, + apppackageincrementversionnumber, + apppackagetoCopy, + apppackagenotToCopy, + apppackagebinariesFolders) = artifactDetails.get(primaryArtifactKey) - def INST_VERBOSE = '' - if (VERBOSE == "true") { - INST_VERBOSE = '-v' - } + def APP_PACKAGE_BIN = "${apppackagedestFolder}" - def INST_OUT = "${INTERMEDIATEEXPORTSPATH}\\${processingFolder}" - fileOperations([folderCreateOperation(INST_OUT)]) + def controlArtifactKey = 'ControlPackageSet' + (controlpackagesourceProject, + controlpackagesourceselector, + controlpackagedestFolder, + controlpackageincrementversionnumber, + controlpackagetoCopy, + controlpackagenotToCopy, + controlpackagebinariesFolders) = artifactDetails.get(controlArtifactKey) - def INST_VARIANT = '' - if (VARIANT != "" ) { - INST_VARIANT = '-r ' + VARIANT - } - installerhelper.checkISMfile("${EXECUTIONPATH}\\${ISMPROJECT}") - installerhelper.selectSigningCertificate(NODE_NAME, "${EXECUTIONPATH}\\${ISMPROJECT}") - - dir(EXECUTIONPATH) { - // INSTALLSHIELD201X_CMD and DOTNET_FRAMEWORK64_PATH must be externally defined - bat """ - "${installcmd}" -t "${DOTNET_FRAMEWORK64_PATH}" -p "${ISMPROJECT}" -y "${INST_ISVERSION}" -l "WORKSPACE=${WORKSPACE}" -l "INST_USB_PMB_C8_3_1040_1041=${INST_USB_PMB_C8_3_1040_1041}" -l "INST_USB_PMB_C8_3_1030=${INST_USB_PMB_C8_3_1030}" -l "INST_USB_PMB_C8_C2=${INST_USB_PMB_C8_C2}" -l "INST_BIN=${INST_BIN}" -l "INST_CONFIGS=${INST_CONFIGS}" -l "INST_RELEASE_NOTES=${INST_RELEASE_NOTES}" -l "INST_RESOURCEFILES=${INST_RESOURCEFILES}" -l "INST_SRC=${INST_SRC}" -l "INST_OUT=${INST_OUT}" -l "INST_EULA=${INST_EULA}" -l "INST_ICCPROFILES=${INST_ICCPROFILES}" -l "INST_SUPPORT=${INST_SUPPORT}" -l "INST_SEC=${INST_SEC}" -l "INST_VERSIONINFO=${VERSIONINFO}" -prqpath "${INST_PREQ}" ${INST_VERBOSE} ${INST_VARIANT} -a x64 - """ - } - - installerhelper.copyToFinalPosition(pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) - - def EXPORT_OUT_GLOB = "${FINALBASEFOLDER}/**" - echo "Adding ${EXPORT_OUT_GLOB} to installer publish list" - installerpublishlist.add(EXPORT_OUT_GLOB) + def CONTROL_PACKAGE_BIN = "${controlpackagedestFolder}" + + def LNPU_BIN = LNPUBUILDIMPORT + + def INST_VERBOSE = '' + if (VERBOSE == "true") { + INST_VERBOSE = '-v' } + + def INST_OUT = "${INTERMEDIATEEXPORTSPATH}\\${processingFolder}" + fileOperations([folderCreateOperation(INST_OUT)]) + + def INST_VARIANT = '' + if (VARIANT != "" ) { + INST_VARIANT = '-r ' + VARIANT + } + installerhelper.checkISMfile("${EXECUTIONPATH}\\${ISMPROJECT}") + installerhelper.selectSigningCertificate(NODE_NAME, "${EXECUTIONPATH}\\${ISMPROJECT}") + + dir(EXECUTIONPATH) { + // INSTALLSHIELD201X_CMD and DOTNET_FRAMEWORK64_PATH must be externally defined + bat """ + "${installcmd}" -t "${DOTNET_FRAMEWORK64_PATH}" -p "${ISMPROJECT}" -y "${INST_ISVERSION}" -l "WORKSPACE=${WORKSPACE}" -l "APP_PACKAGE_BIN=${APP_PACKAGE_BIN}" -l "CONTROL_PACKAGE_BIN=${CONTROL_PACKAGE_BIN}" -l "LNPU_BIN=${LNPU_BIN}" -l "INST_OUT=${INST_OUT}" -l "INST_EULA=${INST_EULA}" -l "INST_SUPPORT=${INST_SUPPORT}" -l "INST_VERSIONINFO=${VERSIONINFO}" -prqpath "${INST_PREQ}" ${INST_VERBOSE} ${INST_VARIANT} -a x64 + """ + } + + installerhelper.copyToFinalPosition(pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) + + def EXPORT_OUT_GLOB = "${FINALBASEFOLDER}/**" + echo "Adding ${EXPORT_OUT_GLOB} to installer publish list" + installerpublishlist.add(EXPORT_OUT_GLOB) } } } stage('Classifying publishable artifacts') { - when { - expression { - toBuild - } - } steps { script { def publishable = installerpublishlist @@ -278,11 +247,6 @@ pipeline { } } stage('Archiving artifacts') { - when { - expression { - toBuild - } - } steps { script { ArrayList itemsForRelease = ["info/**", "installers/**"] @@ -293,11 +257,6 @@ pipeline { } } stage('Check for unstable build') { - when { - expression { - toBuild - } - } steps { script { def installersToBuild = installerDetails.size() From 53b39cece8cd0a0f3a9f72bdaa4e2686251d1308 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 16:23:57 +0000 Subject: [PATCH 45/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 1568e93..80513d3 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -46,7 +46,7 @@ pipeline { exit(1) } - toBuild = ['LNPU'] + toBuild = 'LNPU' } } } From 32649d5bea7bda4303e43d09ac3879c00241bdaa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 16:28:00 +0000 Subject: [PATCH 46/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 2 +- install/install.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 77b917c..6763cb1 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -23,7 +23,7 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, artifactDetails, installerDetails, boolean showdetails = false) { +def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { artifactFetched = [:] if (showdetails) { diff --git a/install/install.groovy b/install/install.groovy index 80513d3..8ece4b5 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -153,7 +153,7 @@ pipeline { LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") - artifactFetched = installerhelper.artifactsImporter(meta, importer, artifactDetails, installerDetails, true) + artifactFetched = installerhelper.artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, true) } } } From cf0c32b0c115a3c4160edca57ee0c914d4de629a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 16:30:27 +0000 Subject: [PATCH 47/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 8ece4b5..1f82630 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -46,7 +46,7 @@ pipeline { exit(1) } - toBuild = 'LNPU' + toBuild = ['LNPU'] } } } From 458bfbde4e0f952336d4f9859da4c15af0ead1a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 16:32:56 +0000 Subject: [PATCH 48/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 1f82630..f3f028e 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -177,7 +177,7 @@ pipeline { script { fileOperations([folderCreateOperation(INTERMEDIATEEXPORTSPATH)]) installerpublishlist = [] - (installcmd, artifactKeySet, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(toBuild) + (installcmd, artifactKeySet, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(toBuild[0]) def primaryArtifactKey = 'AppPackageSet' def appimportmeta = artifactFetched.get(primaryArtifactKey) From ea79baa8832c43b509324104b9d9b118c923b5af Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 16:35:11 +0000 Subject: [PATCH 49/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index f3f028e..5e98a57 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -182,7 +182,7 @@ pipeline { def primaryArtifactKey = 'AppPackageSet' def appimportmeta = artifactFetched.get(primaryArtifactKey) def INST_ISVERSION = appimportmeta.buildstampinfo.buildstamp - echo "For: ${installer} build stamp is: ${INST_ISVERSION}" + echo "Build stamp is: ${INST_ISVERSION}" (apppackagesourceProject, apppackagesourceselector, From f94fcb8b359f6512340261510206368591c8b4a2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 18:46:33 +0000 Subject: [PATCH 50/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 6763cb1..57fe65b 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -111,6 +111,7 @@ def checkISMfilebackend(String xml) { def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) + echo "ISM file xml: ${xml}" (exitcode, message) = checkISMfilebackend(xml) echo message if (exitcode > 0) { From 4ff32610384aa4c5f176c03cbd2770e6d8ec01e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 19:00:14 +0000 Subject: [PATCH 51/73] CORE-594 - Debug installer creation --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 5e98a57..3e6e919 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -205,7 +205,7 @@ pipeline { def CONTROL_PACKAGE_BIN = "${controlpackagedestFolder}" - def LNPU_BIN = LNPUBUILDIMPORT + def LNPU_BIN = LNPUBUILD def INST_VERBOSE = '' if (VERBOSE == "true") { From 5772f192474d2290ea16784d8a89b63a0fd839a3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 21:03:52 +0000 Subject: [PATCH 52/73] CORE-594 - Debug installer creation --- install/install.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 3e6e919..1c1a40e 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -143,14 +143,15 @@ pipeline { lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) def BUILD_STAMP=meta.buildstampinfo.buildstamp + def PRODUCT_NAME=meta.productinfo.productname artifactDetails = [ // Key, Artifacts Source project Artifacts source selector, Destination folder Increment Version Number Artifacts to copy Artifacts to not copy Binaries Folders AppPackageSet: [PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, "${IMPORTSPATH}\\build\\apppackage", false, '**', '**/*.pdb,**/*.lib', [ 'bin', 'scripts'], ], ControlPackageSet: [CONTROL_PACKAGE_CREATE_JOB, CONTROL_PACKAGE_CREATE_JOB_SELECTOR, "${IMPORTSPATH}\\build\\controlpackage", false, '**', '**/*.lib', [ 'bin', 'scripts'], ], ] installerDetails = [ - // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename - LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/XR1003/DiskImages/DISK1/GIS Inkjet OS 2.exe", "${EXPORTSPATH}/XR1003", "GIS Inkjet OS XR1003 ${BUILD_STAMP}.exe" ], + // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename + LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/LNPU.exe", "${EXPORTSPATH}/ALL", "Local Network Package Updater ${PRODUCT_NAME} ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, true) From 0852aed1da0acae2040eac5d584e46d1c7dbef90 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 22:14:00 +0000 Subject: [PATCH 53/73] Squashed 'Common_scripts/' changes from 2c26b22..6a415af 6a415af CORE-594 - Debug installer creation cdd00fd CORE-594 - Debug installer creation eda4801 CORE-594 - Debug installer creation fb721bb CORE-594 - Debug installer creation f1987cb CORE-594 - Debug installer creation d282099 CORE-594 - Debug installer creation 05a32d0 CORE-594 - Debug installer creation d70f633 CORE-594 - Debug installer creation 945f837 CORE-594 - Debug installer creation 36ed63e CORE-594 - Debug installer creation a1c2d3b CORE-594 - Debug installer creation 5b3f7a7 CORE-594 - Debug installer creation cfcdf5c CORE-594 - Debug installer creation f821da7 CORE-594 - Debug installer creation git-subtree-dir: Common_scripts git-subtree-split: 6a415afe26eaa813ef1a5df891b9cfc52f1ee2ce --- import/importer.groovy | 35 ++++++++++++++++++++++++++++++++--- install/installhelper.groovy | 25 ++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index bb0d32e..4948cad 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -20,6 +20,8 @@ class ImportOptions { String m_repo String m_revision + + boolean m_showdetails ImportOptions(script, String sourceselect = 'Jenkins job artifact', @@ -31,7 +33,8 @@ class ImportOptions { boolean flatten = false, boolean includemetainfilter = true, boolean clean = true, - appendtoparentmeta = true) { + appendtoparentmeta = true, + showdetails = false) { m_script = script m_sourceselect = sourceselect.trim() m_incrementversionnumber = incrementversionnumber @@ -43,6 +46,25 @@ class ImportOptions { m_includemetainfilter = includemetainfilter m_clean = clean m_appendtoparentmeta = appendtoparentmeta + m_showdetails = showdetails + } + + String show() { + return """m_incrementversionnumber: ${m_incrementversionnumber} + m_filter: ${m_filter} + m_excludes: ${m_excludes} + m_fingerprint: ${m_fingerprint} + m_nested: ${m_nested} + m_flatten: ${m_flatten} + m_includemetainfilter: ${m_includemetainfilter} + m_clean: ${m_clean} + m_appendtoparentmeta: ${m_appendtoparentmeta} + m_sourceselect: ${m_sourceselect} + m_jobname: ${m_jobname} + m_buildnumberorselector: ${m_buildnumberorselector} + m_repo: ${m_repo} + m_revision: ${m_revision} + """ } void setsourceselect(String sourceselect) { @@ -96,7 +118,7 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void jenkinsjob(String jobname, buildnumberorselector) { + void jenkinsjob(String jobname, String buildnumberorselector) { m_jobname = jobname.trim() m_buildnumberorselector = [ $class: 'ParameterizedBuildSelector', @@ -109,7 +131,7 @@ class ImportOptions { m_jobname = jobname.trim() m_buildnumberorselector = [ $class: 'ParameterizedBuildSelector', - parameterName: "${buildnumberorselector}" + parameterName: "${buildnumberorselector}" ] setsourceselect('Jenkins job artifact') } @@ -125,6 +147,10 @@ class ImportOptions { releaserepo(repo, revision) setsourceselect(sourceselect) } + + void showdetails(boolean showdetails) { + m_showdetails = showdetails + } } ImportOptions getNewImportOptions() { @@ -263,6 +289,9 @@ def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { } def artifactimport(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_showdetails) { + echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" + } switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 96a8b2e..57fe65b 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -23,17 +23,32 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { artifactFetched = [:] + if (showdetails) { + echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") + } def importoptions = importer.getNewImportOptions() + importoptions.showdetails(showdetails) for (installer in toBuild) { - def artifactKeys = installerDetails.get(installer)[1] + def installerinfo = installerDetails.get(installer) + if (showdetails) { + echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") + } + def artifactKeys = installerinfo[1] if (artifactKeys instanceof java.lang.String) { artifactKeys = [ artifactKeys ] } + if (showdetails) { + echo ("artifactKeys:\n${artifactKeys}") + } for (artifactKey in artifactKeys) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + artifactDetail = artifactDetails.get(artifactKey) + if (showdetails) { + echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") + } + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail if (artifactFetched.get(artifactKey) == null) { importoptions.jenkinsjob(sourceProject, sourceselector) importoptions.setincrementversionnumber(incrementversionnumber) @@ -45,6 +60,9 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { } } } + if (showdetails) { + echo ("artifactFetched:\n${artifactFetched}") + } return artifactFetched } @@ -93,6 +111,7 @@ def checkISMfilebackend(String xml) { def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) + echo "ISM file xml: ${xml}" (exitcode, message) = checkISMfilebackend(xml) echo message if (exitcode > 0) { From 8d85badd4af681d4bd2f3a7f1e7db54d9b32255d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 22:17:14 +0000 Subject: [PATCH 54/73] CORE-594 - Debug installer creation --- Common_scripts/install/installhelper.groovy | 2 +- install/install.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy index 57fe65b..8dbf23e 100644 --- a/Common_scripts/install/installhelper.groovy +++ b/Common_scripts/install/installhelper.groovy @@ -111,7 +111,7 @@ def checkISMfilebackend(String xml) { def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) - echo "ISM file xml: ${xml}" + //echo "ISM file xml: ${xml}" (exitcode, message) = checkISMfilebackend(xml) echo message if (exitcode > 0) { diff --git a/install/install.groovy b/install/install.groovy index 1c1a40e..37b1641 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -154,7 +154,7 @@ pipeline { LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/LNPU.exe", "${EXPORTSPATH}/ALL", "Local Network Package Updater ${PRODUCT_NAME} ${BUILD_STAMP}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") - artifactFetched = installerhelper.artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, true) + artifactFetched = installerhelper.artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails) } } } From 514fb12f1c2972f26cd37721ab231031f4ade4b4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 22:18:09 +0000 Subject: [PATCH 55/73] Squashed 'Common_scripts/' changes from 6a415af..951032d 951032d CORE-594 - Debug installer creation git-subtree-dir: Common_scripts git-subtree-split: 951032de5a24f774f294d2241e67ad4260b9301a --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 57fe65b..8dbf23e 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -111,7 +111,7 @@ def checkISMfilebackend(String xml) { def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) - echo "ISM file xml: ${xml}" + //echo "ISM file xml: ${xml}" (exitcode, message) = checkISMfilebackend(xml) echo message if (exitcode > 0) { From b6d72e7e18b38ec804e42baa2c335c18b29fb99a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 22:29:41 +0000 Subject: [PATCH 56/73] CORE-594 - Debug installer creation --- build/build_all.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/build_all.groovy b/build/build_all.groovy index 8f02dd0..ff91fa1 100644 --- a/build/build_all.groovy +++ b/build/build_all.groovy @@ -20,9 +20,8 @@ pipeline { SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}".trim() - SOURCEROOTDIR = 'lnpu' SOURCEDIR = "${WORKSPACE}/lnpu" - configurationinfo = configurationhelper.gitCheckoutAndRegister(meta, SOURCEROOTDIR, SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + configurationinfo = configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) sourcesha = configurationinfo.configurationSHA } } From 1b3435ca534143caacc2d67cafbb8915f3ae9cb6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 13 Jan 2022 23:35:11 +0000 Subject: [PATCH 57/73] CORE-594 - Debug installer creation --- build/build_all.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build_all.groovy b/build/build_all.groovy index ff91fa1..3fa4258 100644 --- a/build/build_all.groovy +++ b/build/build_all.groovy @@ -21,7 +21,7 @@ pipeline { SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' SOURCEREVISION = "${SOURCE_REVISION}".trim() SOURCEDIR = "${WORKSPACE}/lnpu" - configurationinfo = configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + configurationinfo = configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) sourcesha = configurationinfo.configurationSHA } } From a0383e73910eae2e8838f98f15e44727966c54c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Jan 2022 16:08:00 +0000 Subject: [PATCH 58/73] CORE-594 - Debug installer creation --- install/install.groovy | 52 +++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 37b1641..ac5caac 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -103,6 +103,8 @@ pipeline { steps { script { env.IMPORTSPATH = "${WORKSPACE}\\imports" + env.APPPACKAGEIMPORTPATH = "${IMPORTSPATH}\\build\\apppackage" + env.CONTROLPACKAGEIMPORTPATH = "${IMPORTSPATH}\\build\\controlpackage" env.INTERMEDIATEEXPORTSPATH = "${WORKSPACE}\\info" env.EXPORTSPATH = 'installers' env.VERSIONINFO = "${WORKSPACE}\\versioninfo" @@ -140,18 +142,31 @@ pipeline { lnpubuildjobartifactinfo['Release repo'], lnpubuildjobartifactinfo['Release repo revision']) lnpubuildimportoptions.setincrementversionnumber(false) + lnpubuildimportoptions.setexcludes( '**/*.pdb,**/*.lib') lnpubuildmeta = importer.artifactimport(meta, lnpubuildimportoptions, LNPUBUILDIMPORT) - def BUILD_STAMP=meta.buildstampinfo.buildstamp - def PRODUCT_NAME=meta.productinfo.productname + def apppackageimportoptions = importer.getNewImportOptions() + apppackageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, + PACKAGE_CREATE_JOB_SELECTOR) + apppackageimportoptions.setincrementversionnumber(false) + apppackagemeta = importer.artifactimport(meta, apppackageimportoptions, APPPACKAGEIMPORTPATH) + + def controlpackageimportoptions = importer.getNewImportOptions() + controlpackageimportoptions.jenkinsjob(CONTROL_PACKAGE_CREATE_JOB, + CONTROL_PACKAGE_CREATE_JOB_SELECTOR) + controlpackageimportoptions.setincrementversionnumber(false) + controlpackagemeta = importer.artifactimport(meta, controlpackageimportoptions, CONTROLPACKAGEIMPORTPATH) + + BUILD_STAMP = apppackagemeta.buildstampinfo.buildstamp + PRODUCT_NAME = apppackagemeta.productinfo.productname + EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${BUILD_STAMP} LNP Updater" + artifactDetails = [ // Key, Artifacts Source project Artifacts source selector, Destination folder Increment Version Number Artifacts to copy Artifacts to not copy Binaries Folders - AppPackageSet: [PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, "${IMPORTSPATH}\\build\\apppackage", false, '**', '**/*.pdb,**/*.lib', [ 'bin', 'scripts'], ], - ControlPackageSet: [CONTROL_PACKAGE_CREATE_JOB, CONTROL_PACKAGE_CREATE_JOB_SELECTOR, "${IMPORTSPATH}\\build\\controlpackage", false, '**', '**/*.lib', [ 'bin', 'scripts'], ], ] installerDetails = [ - // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename - LNPU: [INSTALLSHIELD2020_CMD, ['AppPackageSet','ControlPackageSet'], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/LNPU.exe", "${EXPORTSPATH}/ALL", "Local Network Package Updater ${PRODUCT_NAME} ${BUILD_STAMP}.exe" ], + // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename + LNPU: [INSTALLSHIELD2020_CMD, [], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/LNPU.exe", "${EXPORTSPATH}/ALL", "${EXE_NAME_PREFIX}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails) @@ -180,31 +195,12 @@ pipeline { installerpublishlist = [] (installcmd, artifactKeySet, processingFolder, EXECUTIONPATH, ISMPROJECT, VARIANT, pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) = installerDetails.get(toBuild[0]) - def primaryArtifactKey = 'AppPackageSet' - def appimportmeta = artifactFetched.get(primaryArtifactKey) - def INST_ISVERSION = appimportmeta.buildstampinfo.buildstamp + def INST_ISVERSION = BUILD_STAMP echo "Build stamp is: ${INST_ISVERSION}" - (apppackagesourceProject, - apppackagesourceselector, - apppackagedestFolder, - apppackageincrementversionnumber, - apppackagetoCopy, - apppackagenotToCopy, - apppackagebinariesFolders) = artifactDetails.get(primaryArtifactKey) + def APP_PACKAGE_BIN = APPPACKAGEIMPORTPATH - def APP_PACKAGE_BIN = "${apppackagedestFolder}" - - def controlArtifactKey = 'ControlPackageSet' - (controlpackagesourceProject, - controlpackagesourceselector, - controlpackagedestFolder, - controlpackageincrementversionnumber, - controlpackagetoCopy, - controlpackagenotToCopy, - controlpackagebinariesFolders) = artifactDetails.get(controlArtifactKey) - - def CONTROL_PACKAGE_BIN = "${controlpackagedestFolder}" + def CONTROL_PACKAGE_BIN = CONTROLPACKAGEIMPORTPATH def LNPU_BIN = LNPUBUILD From c84a6cc5bee762502e189812fac8af7fcdf7fc9c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Jan 2022 16:56:52 +0000 Subject: [PATCH 59/73] Squashed 'Common_scripts/' changes from 951032d..2ab8282 2ab8282 Merge commit 'ea3a620531419dd99f369c664f7fb04291dfe2c0' into master 8e15e7b CORE-594 - Import artifacts using full name git-subtree-dir: Common_scripts git-subtree-split: 2ab828273c52b8e1d4c948659f3962cbcaa7b29b --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 8e5e56e..a40e9a8 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,7 +4,7 @@ def combineSubjobArtifacts(meta, importer, subJobs) { for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { - def subJobName = subJobObject.getProjectName() + def subJobName = subJobObject.getFullProjectName() def subJobNumber = subJobObject.getNumber() def destFolder = subJob[1] def boolean incrementversionnumber = subJob[2] From 6bcf0ea65a4fe27306b0f0a727dbc42dc954d9a0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Jan 2022 18:49:29 +0000 Subject: [PATCH 60/73] CORE-594 - Use an ini file to pass parameters to install shield --- install/install.groovy | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index ac5caac..c2090d6 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -219,12 +219,42 @@ pipeline { installerhelper.checkISMfile("${EXECUTIONPATH}\\${ISMPROJECT}") installerhelper.selectSigningCertificate(NODE_NAME, "${EXECUTIONPATH}\\${ISMPROJECT}") + // Fill settings file + //-i <.ini file path> + def IS_INI_CONTENT = [ + "[Project]", + "Name=\"${ISMPROJECT}\"", + "BuildLabel=x64", + "Product=\"${EXE_NAME_PREFIX}\"", + "", + "[Release]", + "Name=${VARIANT}", + "", + "[Mode]", + "Verbose=${(VERBOSE == "true")?'yes':'no'}", + "DotNetPath=\"${DOTNET_FRAMEWORK64_PATH}\"", + "PrerequisitePath=\"${INST_PREQ}\"", + ].join('\n') + dir(EXECUTIONPATH) { - // INSTALLSHIELD201X_CMD and DOTNET_FRAMEWORK64_PATH must be externally defined + def IS_INI_FILE = 'is.ini' + echo "Writing:\n${IS_INI_CONTENT}\nto: ${IS_INI_FILE}" + writeFile(file: IS_INI_FILE, text: IS_INI_CONTENT) bat """ - "${installcmd}" -t "${DOTNET_FRAMEWORK64_PATH}" -p "${ISMPROJECT}" -y "${INST_ISVERSION}" -l "WORKSPACE=${WORKSPACE}" -l "APP_PACKAGE_BIN=${APP_PACKAGE_BIN}" -l "CONTROL_PACKAGE_BIN=${CONTROL_PACKAGE_BIN}" -l "LNPU_BIN=${LNPU_BIN}" -l "INST_OUT=${INST_OUT}" -l "INST_EULA=${INST_EULA}" -l "INST_SUPPORT=${INST_SUPPORT}" -l "INST_VERSIONINFO=${VERSIONINFO}" -prqpath "${INST_PREQ}" ${INST_VERBOSE} ${INST_VARIANT} -a x64 + "${installcmd}" ^ + -i "${IS_INI_FILE}" ^ + -y "${INST_ISVERSION}" ^ + -l "WORKSPACE=${WORKSPACE}" ^ + -l "APP_PACKAGE_BIN=${APP_PACKAGE_BIN}" ^ + -l "CONTROL_PACKAGE_BIN=${CONTROL_PACKAGE_BIN}" ^ + -l "LNPU_BIN=${LNPU_BIN}" ^ + -l "INST_OUT=${INST_OUT}" ^ + -l "INST_EULA=${INST_EULA}" ^ + -l "INST_SUPPORT=${INST_SUPPORT}" ^ + -l "INST_VERSIONINFO=${VERSIONINFO}" """ } + //-z installerhelper.copyToFinalPosition(pathToIntermediate, FINALBASEFOLDER, FINALFILENAME) From 8e00a4b0eb4f40aea277bb7f8f79ef506c1f428a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Jan 2022 17:31:03 +0000 Subject: [PATCH 61/73] CORE-594 - Try to customise the Product Name property using the -z parameter --- compile/compile.groovy | 2 +- install/install.groovy | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/compile/compile.groovy b/compile/compile.groovy index cedf888..78ba926 100644 --- a/compile/compile.groovy +++ b/compile/compile.groovy @@ -32,7 +32,7 @@ pipeline { steps { script { SOURCEURL = 'git@hydra:core-presto/pc-utilities/local-network-package-updater.git' - SOURCEREVISION = "${SOURCE_REVISION}" + SOURCEREVISION = SOURCE_REVISION SOURCE_PATH = 'lnpu' configurationhelper.gitCheckoutAndRegister(meta, 'lnpu', SOURCEURL, SOURCEREVISION, SOURCE_PATH, '', 0, false) env.SRC = "${SOURCE_PATH}\\Local Network Package Updater" diff --git a/install/install.groovy b/install/install.groovy index c2090d6..212fea3 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -216,8 +216,17 @@ pipeline { if (VARIANT != "" ) { INST_VARIANT = '-r ' + VARIANT } - installerhelper.checkISMfile("${EXECUTIONPATH}\\${ISMPROJECT}") - installerhelper.selectSigningCertificate(NODE_NAME, "${EXECUTIONPATH}\\${ISMPROJECT}") + + def ISM_PROJECT_FILE_PATH = "${EXECUTIONPATH}\\${ISMPROJECT}" + convertdetails = [ + // File to convert Pattern to replace New string + [ ISM_PROJECT_FILE_PATH, "-INSTALLER PRODUCT NAME-", EXE_NAME_PREFIX ], + ] + versionnumbering = load("${autobuilder.commonscripts()}/versionnumbering/do.groovy") + //versionnumbering.stamp(convertdetails) + + installerhelper.checkISMfile(ISM_PROJECT_FILE_PATH) + installerhelper.selectSigningCertificate(NODE_NAME, ISM_PROJECT_FILE_PATH) // Fill settings file //-i <.ini file path> @@ -251,7 +260,8 @@ pipeline { -l "INST_OUT=${INST_OUT}" ^ -l "INST_EULA=${INST_EULA}" ^ -l "INST_SUPPORT=${INST_SUPPORT}" ^ - -l "INST_VERSIONINFO=${VERSIONINFO}" + -l "INST_VERSIONINFO=${VERSIONINFO}" ^ + -z "ProductName=${EXE_NAME_PREFIX}" """ } //-z From e4b734dc04bc1a1f1b38395898fb255ae411a304 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Jan 2022 11:35:52 +0000 Subject: [PATCH 62/73] CORE-549 - Perform string substitions on ism file - to make the installer unique --- install/install.groovy | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 212fea3..fd5e5af 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -157,7 +157,7 @@ pipeline { controlpackageimportoptions.setincrementversionnumber(false) controlpackagemeta = importer.artifactimport(meta, controlpackageimportoptions, CONTROLPACKAGEIMPORTPATH) - BUILD_STAMP = apppackagemeta.buildstampinfo.buildstamp + BUILD_STAMP = apppackagemeta.globalreleasetag() PRODUCT_NAME = apppackagemeta.productinfo.productname EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${BUILD_STAMP} LNP Updater" @@ -217,10 +217,20 @@ pipeline { INST_VARIANT = '-r ' + VARIANT } + def nextproductcode = UUID.randomUUID().toString() + def nextupgradecode = UUID.randomUUID().toString() + def ISM_PROJECT_FILE_PATH = "${EXECUTIONPATH}\\${ISMPROJECT}" convertdetails = [ - // File to convert Pattern to replace New string - [ ISM_PROJECT_FILE_PATH, "-INSTALLER PRODUCT NAME-", EXE_NAME_PREFIX ], + // File to convert Pattern to replace New string + // [ ISM_PROJECT_FILE_PATH, '-INSTALLER PRODUCT NAME-', EXE_NAME_PREFIX ], // Use -z for this + [ ISM_PROJECT_FILE_PATH, '-INSTALLER PRODUCT NAME-', EXE_NAME_PREFIX ], + [ ISM_PROJECT_FILE_PATH, '-INSTALLER PROGRAM DIRECTORY-', EXE_NAME_PREFIX ], + [ ISM_PROJECT_FILE_PATH, '-INSTALLER SETUP FILENAME-', EXE_NAME_PREFIX ], + [ ISM_PROJECT_FILE_PATH, '{8E581C6A-A296-4C45-90E9-6F36C1465590}', nextproductcode ], + [ ISM_PROJECT_FILE_PATH, '{22047ECF-8BBF-4A1E-9D40-1EEA1B77B0F2}', nextupgradecode ], + [ ISM_PROJECT_FILE_PATH, '-INSTALLER SUBJECT-', "Installer for updating ${PRODUCT_NAME} firmware" ], + [ ISM_PROJECT_FILE_PATH, 'Launch LNPU', "Launch ${EXE_NAME_PREFIX}" ], ] versionnumbering = load("${autobuilder.commonscripts()}/versionnumbering/do.groovy") //versionnumbering.stamp(convertdetails) From 85f8e28d7020c138ceaeaa069ab49bd557f95cdf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Jan 2022 11:37:29 +0000 Subject: [PATCH 63/73] CORE-549 - Perform string substitions on ism file - to make the installer unique --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index fd5e5af..2070ef4 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -166,7 +166,7 @@ pipeline { ] installerDetails = [ // Key, Installer command artifactSet Target folder Execution path ISM project file Variant Intermediate output path Component base folder Final filename - LNPU: [INSTALLSHIELD2020_CMD, [], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/LNPU.exe", "${EXPORTSPATH}/ALL", "${EXE_NAME_PREFIX}.exe" ], + LNPU: [INSTALLSHIELD2020_CMD, [], 'install_core', "${SOURCE_PATH}\\installer\\build\\core", 'Core.ism', '', "${INTERMEDIATEEXPORTSPATH}/install_core/x64/ALL/DiskImages/DISK1/${EXE_NAME_PREFIX}.exe", "${EXPORTSPATH}/ALL", "${EXE_NAME_PREFIX}.exe" ], ] installerhelper = load("${autobuilder.commonscripts()}/install/installhelper.groovy") artifactFetched = installerhelper.artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails) From 433c860b02cee71a83355c92c043ad3e5b3d421e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Jan 2022 11:53:16 +0000 Subject: [PATCH 64/73] CORE-549 - Perform string substitions on ism file - to make the installer unique --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 2070ef4..33eb3a7 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -233,7 +233,7 @@ pipeline { [ ISM_PROJECT_FILE_PATH, 'Launch LNPU', "Launch ${EXE_NAME_PREFIX}" ], ] versionnumbering = load("${autobuilder.commonscripts()}/versionnumbering/do.groovy") - //versionnumbering.stamp(convertdetails) + versionnumbering.stamp(convertdetails) installerhelper.checkISMfile(ISM_PROJECT_FILE_PATH) installerhelper.selectSigningCertificate(NODE_NAME, ISM_PROJECT_FILE_PATH) From f0820158180822521f89fa5b9736640eb8bea4b0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Jan 2022 12:05:57 +0000 Subject: [PATCH 65/73] CORE-549 - Perform string substitions on ism file - to make the installer unique --- install/install.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install.groovy b/install/install.groovy index 33eb3a7..e2496e3 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -157,7 +157,7 @@ pipeline { controlpackageimportoptions.setincrementversionnumber(false) controlpackagemeta = importer.artifactimport(meta, controlpackageimportoptions, CONTROLPACKAGEIMPORTPATH) - BUILD_STAMP = apppackagemeta.globalreleasetag() + BUILD_STAMP = apppackagemeta.releasetag() PRODUCT_NAME = apppackagemeta.productinfo.productname EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${BUILD_STAMP} LNP Updater" From d1a56a994c420a77f8bbf1909d510366a1578f8c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 1 Feb 2022 14:55:48 +0000 Subject: [PATCH 66/73] Squashed 'Common_scripts/' changes from 2ab8282..0326879 0326879 Eliminate a copy for performance cbc3289 Eliminate a copy for performance 3271d90 Eliminate a copy for performance 4044992 Eliminate a copy for performance 2e74e8a Eliminate a copy for performance 10d0019 Ensure script objects are around when source files have been cleaned 1e29bd5 Force item copier workspace to be cleaned each time 5575f2b Don't attempt to import artifacts associated with components that do not have complete meta 9b89c0f CORE-348 - Reverse a debugging change 3db76ce Merge remote-tracking branch 'origin/master' into CORE-348 2a17eb0 Merge commit 'ad8723e7c8df7b9ffceec3540fd88e9f847e7a7f' into production b9f5240 / missing when doing hierarchical release repo checkout e67bec1 When doing nested release repo imports - ignore components that have incomplete release info 16d7c9c Merge remote-tracking branch 'origin/master' into CORE-348 56951ba CORE-348: Trying to resolve a git error in the autobuild. git-subtree-dir: Common_scripts git-subtree-split: 0326879391035840f42f5e1eb5ffc92e834c8995 --- copy/all.groovy | 36 ++++++++++++------------------------ import/importer.groovy | 20 +++++++++++--------- 2 files changed, 23 insertions(+), 33 deletions(-) diff --git a/copy/all.groovy b/copy/all.groovy index aab479c..24aacfb 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -4,21 +4,13 @@ pipeline { stage('Initial setup') { steps { script { + // Force workspace to be cleaned + CLEAN_WORKSPACE = "true" meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean exports') { - steps { - script { - IMPORTS = 'imports' - cleanfolders = [ - IMPORTS, - ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") } } } @@ -34,29 +26,25 @@ pipeline { SOURCE_ARTIFACT_RELEASE_REVISION) echo "Nested: ${NESTED_COPY}" copyoptions.setnested(NESTED_COPY == 'true') - importer.artifactimport(meta, copyoptions, IMPORTS) + importer.artifactimport(meta, copyoptions, "${WORKSPACE}/.") } } } stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(['**/*']) - def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + // Save the imported meta - while the unwanted meta is written to file then deleted + def stashname = 'Imported meta stash' + stash(allowEmpty: true, + includes: 'meta/**', + name: stashname, + useDefaultExcludes: false) + def artifactlist = meta.generateartifacts(['**/*']) cleaner.clean(['meta', 'autobuild']) - dir(IMPORTS) { - fileOperations([ - fileCopyOperation(excludes: '', - flattenFiles: false, - includes: '**/*', - targetLocation: WORKSPACE) - ]) - } + unstash(name: stashname) - cleaner.clean([IMPORTS]) - artifactarchiver.archive(meta, artifactlist) } } diff --git a/import/importer.groovy b/import/importer.groovy index 4948cad..fa32ebf 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -239,15 +239,17 @@ def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String de def cloneurl = relatedmeta.releaseinfo.releaserepoURL def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" - echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - - gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) - - gh.stripgithistory(clonepath) - - def metafilepath = "${clonepath}${relatedmeta.metafilepath()}" - echo "Marking related meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(relatedmeta, metafilepath) + if (relatedmeta.componentidentifier() != '_' && + cloneurl != '') { + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } else { + echo "Not importing component that would be placed at path: ${clonepath}" + } } gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" From ab3516d23dbe1987e910de8ff63f6e8732bcc52d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 2 Mar 2022 13:29:36 +0000 Subject: [PATCH 67/73] Squashed 'Common_scripts/' changes from 0326879..b70fe5f b70fe5f Remove debug release filtering 3730461 Debug release filtering db92ee3 Debug release filtering 2393146 Use releasetag script 54bf29f Attempt to separate dependent products and sub-components 3c4de23 Debugging script d93f942 Debugging script caf71e3 Debugging script c993ecc Debugging script 2e5b6b2 Debugging script 0ad763c Debugging script d5a178b Debugging script 01d91a7 Debugging script d2a22db Debugging script 30ac58d Debugging script 490b37b Debugging script 2939eaf Debugging script 35d3862 Debugging script defc807 Debugging script f2e3e52 Debugging script b5e2894 Debugging script c2faf8a Debugging script fa199eb Debugging script d418e5a Debugging script f34fb2e Debugging script 04e947c Debugging script 48d7b0f Debugging script 38b562b Debugging script ac5e7cd Debugging script 1d3f184 Debugging script b187d23 Debugging script 1131865 Debugging script 68cbc00 Debugging script 5771490 Debugging ed17de0 Debugging b1bc540 Debugging e6c6fcf Debugging 57eeaf4 Comment out code to debug later a576d64 Script reorg 32a3aed Supply release tag processing 01ce780 Supply release tag processing af3e22d Supply release tag processing b7c5a66 Supply release tag processing 4fbe1d9 Supply release tag processing 44b1e13 Supply release tag processing af3c041 Supply release tag processing 8b361de Supply release tag processing 89736f0 Supply release tag processing 70d569c Supply release tag processing d2b8d1b Supply release tag processing ce33f59 Supply release tag processing 88cba3b Supply release tag processing 80bc64e Supply release tag processing 6488203 Supply release tag processing c2b27a8 Merge commit 'cd98f833a8db023604d045d9343f18f3a423da5c' 704e499 Start adding dependencies 940a8d5 Start adding dependencies 4a46b8f Build up a tag map 0ca7886 Build up a tag map 3dbdf85 Build up a tag map fb72031 Build up a tag map be599cb Build up a tag map 462b177 Build up a tag map 453a14a Build up a tag map c5f01d0 Build up a tag map 11826a4 Get list of tags for each product release repo 3981ece Filter long name releases from version selection cad2d4e Filter long name releases from version selection 8bd075f Filter long name releases from version selection 20c5648 Merge commit '6633e04d0257e6d52427b2ff761c8a5625196346' as 'Common_scripts' 9a31beb Remove folder 2767e8a Rename folder a5ee31d New files 2c6a90d New files e5d8c38 Merge commit 'c1344e90845db3aa76de5f3d0fc93abb462f8782' as 'Commons_scripts' c1344e9 Squashed 'Commons_scripts/' content from commit 0326879 9171d4d Initial commit e5bca0d Initial commit git-subtree-dir: Common_scripts git-subtree-split: b70fe5f7bd258ba0c3be2eaeebb12403e01c460b --- meta/folder.groovy | 8 ++ product/info.groovy | 217 ++++++++++++++++++++++++++++++++ publish/promptingpublish.groovy | 34 ++--- release/tag.groovy | 39 ++++++ versioncontrol/git.groovy | 23 ++++ 5 files changed, 305 insertions(+), 16 deletions(-) create mode 100644 product/info.groovy create mode 100644 release/tag.groovy diff --git a/meta/folder.groovy b/meta/folder.groovy index 6715145..ac3d0dd 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -913,6 +913,14 @@ class MetaInfo extends GISJsonSerialised { String componentidentifier() { return productinfo.productname + "_" + productinfo.subcomponentname } + + String productvariant() { + if (!productinfo.variant) { + return productinfo.productname + } else { + return productinfo.productname + "_" + productinfo.variant + } + } String componentversionuniqueidentifier() { return componentidentifier() + "_" + buildstampinfo.buildstamp diff --git a/product/info.groovy b/product/info.groovy new file mode 100644 index 0000000..3d7bc30 --- /dev/null +++ b/product/info.groovy @@ -0,0 +1,217 @@ + +importer = load("${autobuilder.commonscripts()}/import/importer.groovy") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +rt = load("${autobuilder.commonscripts()}/release/tag.groovy") + +class ProductDependency { + private script + private rtscript + private String productvariant + private String releaserepo + private String presplitreleasetag + private String releasetag + private String releasestate + + def ProductDependency(script, + rtscript, + String productvariant, + String presplitreleasetag, + String releaserepo) { + this.script = script + this.rtscript = rtscript + this.productvariant = productvariant + this.releaserepo = releaserepo + this.presplitreleasetag = presplitreleasetag + this.releasetag = '' + this.releasestate = '' + } + + def setreleasestate() { + def LinkedHashMap results = rtscript.startswithspecialprefix(presplitreleasetag) + script.echo("results is: ${results.dump()}") + if (results.hasspecialprefix) { + releasetag = results.tagwithoutprefix + releasestate = results.prefixfound + } else { + releasetag = results.tagwithoutprefix + releasestate = '' + } + } +} + +class ProductComponent { + private String componentname + private String releaserepo + private String releasetag + + def ProductComponent(String componentname, String releasetag, String releaserepo) { + this.componentname = componentname + this.releaserepo = releaserepo + this.releasetag = releasetag + } +} + +class ProductRelease { + private String sha + private String builddate + private ArrayList tags + private ArrayList productdependencies + private ArrayList productcomponents + + def ProductRelease(String sha, String builddate, ArrayList tags) { + this.sha = sha + this.builddate = builddate + this.tags = tags + this.productdependencies = new ArrayList() + this.productcomponents = new ArrayList() + } + + def addProductDependency(ProductDependency dependency) { + productdependencies.add(dependency) + } + + def addProductComponent(ProductComponent component) { + productcomponents.add(component) + } +} + +class ProductParsed { + enum State { + UNPARSED, + PARSED, + } + private State scanstate + private String productvariant + private ArrayList releases + + def ProductParsed(String productvariant) { + scanstate = State.UNPARSED + this.productvariant = productvariant + this.releases = new ArrayList() + } + + def markparsed() { + scanstate = State.PARSED + } + + def boolean parsed() { + return (scanstate == State.PARSED) + } + + def addProductRelease(ProductRelease productrelease) { + releases.add(productrelease) + } +} + +def addProductEntry(LinkedHashMap products, + String productvariant, + String releaserepo) { + def productparsed + if (products.get(releaserepo) == null) { // Use the first entry name only + productparsed = new ProductParsed(productvariant) + products.put(releaserepo, productparsed) + } else { + productparsed = products.get(releaserepo) + } + return productparsed +} + +def parseproductdependencies(LinkedHashMap products, + productreleasemeta, + String productreleasefolder, + String containingproductvariant, + ProductRelease productrelease) { + def dependentmeta = productreleasemeta.getassociated(productreleasefolder) + + for (meta in dependentmeta) { + def pd = new ProductDependency(this, + rt, + meta.productvariant(), + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + pd.setreleasestate() + if (meta.productinfo.subcomponentname == 'WHOLE') { + if (meta.productvariant() != containingproductvariant) { // Don't add myself + addProductEntry(products, + meta.productvariant(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } + } else if (meta.productvariant() != containingproductvariant) { + addProductEntry(products, + meta.productvariant(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } else { + def pc = new ProductComponent(meta.productinfo.subcomponentname, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductComponent(pc) + } + } +} + +def parseproductrepo(LinkedHashMap products, + String productrepo, + String productreleasefolder) { + def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) + def String productvariant + def ProductParsed pp + + def ArrayList productreleases = new ArrayList() + shatagmap.each { sha, tags -> + def importoptions = importer.getNewImportOptions() + importoptions.releaserepo(productrepo, sha) + importoptions.setfilter('meta/**') + def productreleasemeta = importer.artifactimport(meta, + importoptions, + productreleasefolder) + + if (!productvariant) { + productvariant = productreleasemeta.productvariant() + pp = addProductEntry(products, + productvariant, productrepo) + } + + def productrelease = new ProductRelease(sha, + productreleasemeta.jenkinsbuildinfo.builddate, + tags) + parseproductdependencies(products, + productreleasemeta, + productreleasefolder, + productvariant, + productrelease) + pp.addProductRelease(productrelease) + } + pp.markparsed() +} + +def iterateinitialproductrepos(initialproductcollection, + LinkedHashMap productcollection, + String productreleasefolder) { + for (productrepo in initialproductcollection.ProductReleaseRepos) { + echo "Querying ${productrepo}" + parseproductrepo(productcollection, + productrepo, + productreleasefolder) + } + while( true ) { + boolean allparsed = true + productcollection.each { releaserepo, productparsed -> + if (!productparsed.parsed()) { + allparsed = false + parseproductrepo(productcollection, + releaserepo, + productreleasefolder) + } + } + if (allparsed) break + } +} + +def LinkedHashMap getNewProductCollection() { + return new LinkedHashMap() +} + + +return this \ No newline at end of file diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index ce6a805..49d9421 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -52,27 +52,29 @@ pipeline { stage('Identify software release to publish') { steps { script { + rt = load("${autobuilder.commonscripts()}/release/tag.groovy") gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) def releases = new ArrayList() - if (FILTER_REDUNDANT_RELEASES == 'false') { - echo("Showing all releases") - releases = allreleases - } else { - echo("Showing filtered releases") - for (release in allreleases) { - if (!release.toLowerCase().startsWith("superceded") && - !release.toLowerCase().startsWith("superseded") && - !release.toLowerCase().startsWith("erroneous") && - !release.toLowerCase().startsWith("faulty") && - !release.toLowerCase().startsWith("ignore") && - !release.toLowerCase().startsWith("inmainline")) { - echo("Showing release: ${release}") - releases.add(release) - } else { - echo("Not showing release: ${release}") + for (release in allreleases) { + def boolean canadd = true + if (FILTER_REDUNDANT_RELEASES == 'true') { + def results = rt.startswithspecialprefix(release) + if (results.prefixfound) { + canadd = false } } + if (FILTER_LONGNAME_RELEASES == 'true') { + if (release.findAll(/[a-zA-Z]/)) { + canadd = false + } + } + if (canadd) { + echo("Showing release: ${release}") + releases.add(release) + } else { + echo("Not showing release: ${release}") + } } timeout(time: 10, unit: 'MINUTES') { release = input( diff --git a/release/tag.groovy b/release/tag.groovy new file mode 100644 index 0000000..6155e7f --- /dev/null +++ b/release/tag.groovy @@ -0,0 +1,39 @@ + +def startswithspecialprefix(String releasetag) { + def RELEASETAG_PREFIXES = [ + 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', + 'superseded' : 'Indicates a later release has superseded the release with this prefix', + 'erroneous' : 'The release was released unintentionally', + 'faulty' : 'The release was found to have a serious fault', + 'ignore' : 'The release should be ignored', + 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', + ] + + def RELEASETAG_PREFIX_SEPERATOR = '_' + + def boolean hasspecialprefix = false + def String releasetagprefix = '' + def String prefixfound = '' + def String prefixmeaning = '' + def String tagwithoutprefix = releasetag + + for (def prefix in RELEASETAG_PREFIXES.keySet()) { + def meaning = RELEASETAG_PREFIXES.get(prefix) + if (releasetag.toLowerCase().startsWith(prefix)) { + prefixfound = prefix + hasspecialprefix = true + prefixmeaning = meaning + tagwithoutprefix = releasetag[(prefix.size() + 1)..-1] + break + } + } + + def results = [ 'hasspecialprefix' : hasspecialprefix, + 'prefixfound' : prefixfound, + 'tagwithoutprefix' : tagwithoutprefix, + 'meaning' : prefixmeaning, + ] + return results +} + +return this diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a22054b..61378b5 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,6 +34,29 @@ ArrayList gettaglist(String workingSetRootPath) { return taglist } +LinkedHashMap> getremoteshatagmap(String repositoryURL) { + def shatagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") + def shataglist = shatagliststring.split('\\n') + + LinkedHashMap> shatagmap = new LinkedHashMap>() + for (shatag in shataglist) { + def shatagfields = shatag.split('\\t') + def sha = shatagfields[0].trim() + def tag = shatagfields[1].trim() + echo "sha: ${sha} tag: ${tag}" + if (shatagmap.get(sha) == null) { + def ArrayList taglist = new ArrayList() + taglist.add(tag) + shatagmap.put(sha, taglist) + } else { + def taglist = shatagmap.get(sha) + taglist.add(tag) + shatagmap.put(sha, taglist) + } + } + return shatagmap +} + @NonCPS def ArrayList sparsecheckoutconfig(ArrayList paths) { ArrayList sparsecheckoutpaths = new ArrayList() From 8d51dc788a05bd4d4329051814986152fcfa11e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 2 Mar 2022 13:38:07 +0000 Subject: [PATCH 68/73] Handle product variants; Change the title of the installer executable --- install/install.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index e2496e3..d5b495e 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -35,7 +35,7 @@ pipeline { stage('Initial setup') { steps { script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO, RELEASE_REPO_VARIANT_SUFFIX) set(LNPU_BINARIES_INFO) lnpubuildjobartifactinfo = get()['Artifact info'] @@ -159,7 +159,7 @@ pipeline { BUILD_STAMP = apppackagemeta.releasetag() PRODUCT_NAME = apppackagemeta.productinfo.productname - EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${BUILD_STAMP} LNP Updater" + EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${RELEASE_REPO_VARIANT_SUFFIX} ${BUILD_STAMP} LNP Updater" artifactDetails = [ // Key, Artifacts Source project Artifacts source selector, Destination folder Increment Version Number Artifacts to copy Artifacts to not copy Binaries Folders From ef15ab68681df6a42674955d4362dcb2444bb664 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 23 Mar 2022 08:19:15 +0000 Subject: [PATCH 69/73] Squashed 'Common_scripts/' changes from b70fe5f..791c94c 791c94c Pass meta to post action handlers; Pass release repo to post actions d21f4b1 Pass meta to post action handlers; Pass release repo to post actions 5e6c640 Merge remote-tracking branch 'origin/master' into production 7c05dde Use productvariant() instead of .productinfo.productname 8cd7ae9 Merge commit '0bb8e883772eb23ba839d6b934857b78fc1f5317' into production 2b61646 Products are those with WHOLE subcomponent 83257ea Merge commit 'a29f25f7be570e0c07a12f35055f8818cef47fa7' into production 062711c Merge commit 'ac6ed3de46417d119c9bd12ca1b1879976f99248' 2ae44ff Use productvariant() instead of .productinfo.productname 2e74b32 Provide a product variant translation table 806f098 Provide a product variant translation table 5db7c8d Provide a product variant translation table 3e317a5 Provide a product variant translation table 58d6990 Provide a product variant translation table ba8062d Provide a product variant translation table 14b1eff Provide a product variant translation table bf4a428 Debug sha/tags table e72f006 Debug sha/tags table 295c18c Debug sha/tags table 8aeb9a9 Debug sha/tags table ab4165e Debug sha/tags table 839f987 Debug sha/tags table cd0a353 Debug sha/tags table 94de9d2 Debug sha/tags table 60c1fba Debug sha/tags table 390c97c Debug sha/tags table da33b16 Debug sha/tags table 9235c1a Debug sha/tags table eb960a6 Debug sha/tags table 9f0a5db Debug sha/tags table 93b2f2b Debug sha/tags table e711a6c Debug sha/tags table 149aad2 Debug sha/tags table fd795d2 Debug sha/tags table 2580ed3 Debug sha/tags table 66a6672 Debug sha/tags table df5c9d1 Remove debug 82f9bb6 Use raw git to checkout each tag 1c9579f Use raw git to checkout each tag f55ca99 Merge commit '610e4be77e08d4b03b8157e38c099c43b2d6834e' into production 7682029 Merge commit 'e8e55db451e17f7b3edaab583e77a0428cebdfe7' into master 1810f08 Merge commit '52965d83743e36da81c13d1b0e98e29244fbc095' 2f496a5 Merge commit '7707b2cffa0e4c308bb6453f5ef3b0eeffc3acf8' 4ea46e3 Merge commit 'dea77dfa289efef365b5d1b48262c089832a5fa3' 85afca9 Create a script for publishing a set of debian packages to a remote update server e408bd1 Create a script for publishing a set of debian packages to a remote update server 6fd010a Create a script for publishing a set of debian packages to a remote update server 7515c39 Create a script for publishing a set of debian packages to a remote update server 0c5fb98 Create a script for publishing a set of debian packages to a remote update server 9ad83e9 Create a script for publishing a set of debian packages to a remote update server 80c2a61 Create a script for publishing a set of debian packages to a remote update server b0cc38c Create a script for publishing a set of debian packages to a remote update server 2c11a0c Create a script for publishing a set of debian packages to a remote update server 7e987f0 Create a script for publishing a set of debian packages to a remote update server f9c8438 Create a script for publishing a set of debian packages to a remote update server 46fc969 Create a script for publishing a set of debian packages to a remote update server b72ddc8 Create a script for publishing a set of debian packages to a remote update server 1beb23e Create a script for publishing a set of debian packages to a remote update server 97b31de Create a script for publishing a set of debian packages to a remote update server 9285f75 Create a script for publishing a set of debian packages to a remote update server d4513a8 Merge remote-tracking branch 'origin/production' b0d92b0 CORE-348 - Provide a script that can obtain a package from a release repo aa124fa CORE-348 - Provide a script that can obtain a package from a release repo 05f9c1b CORE-348 - Provide a script that can obtain a package from a release repo ddd07b9 CORE-348 - Provide a script that can obtain a package from a release repo ae97402 CORE-348 - Build a som-only dependent FDHV package git-subtree-dir: Common_scripts git-subtree-split: 791c94c94d28ee29daf58b55738cd9c13cd158eb --- build/postactions.groovy | 30 ++-- differences/dometahierarchy.groovy | 18 +-- documentation/doxygen_deploy.groovy | 6 +- meta/folder.groovy | 12 +- product/info.groovy | 205 ++++++++++++++++++++-------- publish/do.groovy | 6 +- publish/package.groovy | 36 ++--- publish/packagecollection.groovy | 183 +++++++++++++++++++++++++ release/do.groovy | 8 +- somsoftware/sominfo.groovy | 31 +++++ versioncontrol/git.groovy | 4 +- 11 files changed, 414 insertions(+), 125 deletions(-) create mode 100644 publish/packagecollection.groovy create mode 100644 somsoftware/sominfo.groovy diff --git a/build/postactions.groovy b/build/postactions.groovy index 8b18908..3206004 100644 --- a/build/postactions.groovy +++ b/build/postactions.groovy @@ -85,16 +85,20 @@ def getActionJobName(postbuildactions, String action, String alternativeparamnam return jobname } -def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { +def deployjob(meta, String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername, String releaserepoparametername = '') { def jobname = getActionJobName(postbuildactions, action, alternativeparamname) if (jobname) { + def parameters = [ + new StringParameterValue(jobnameparametername, JOB_NAME), + new StringParameterValue(jobselectorparametername, buildselector), + ] + if (releaserepoparametername) { + parameters += new StringParameterValue(releaserepoparametername, meta.releaseinfo.releaserepoURL) + } build( wait: false, job: jobname, - parameters: [ - string(name: jobnameparametername, value: JOB_NAME), - string(name: jobselectorparametername, value: buildselector), - ] + parameters: parameters ) } } @@ -107,7 +111,7 @@ def readPostBuildInfo() { return postbuildinfo } -def performSuccessfulBuildActions() { +def performSuccessfulBuildActions(meta) { def postbuildactions if (params.POST_BUILD_INFO != null) { def postbuildinfo = readPostBuildInfo() @@ -119,12 +123,12 @@ def performSuccessfulBuildActions() { } } def String buildselector = "${BUILD_NUMBER}" - deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') - deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) - deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') - deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(meta, buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR', 'TOP_LEVEL_META_REPOSITORY') + deployjob(meta, buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') } def sendemails(emailinfo) { @@ -201,7 +205,7 @@ def sendextemail(extmailinfo) { ) } -def performFailedBuildActions() { +def performFailedBuildActions(meta) { def postbuildactions = null if (params.POST_BUILD_INFO != null) { def postbuildinfo = readPostBuildInfo() diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3b25798..f9a2404 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -27,7 +27,7 @@ class JiraTicketInfo { } class DifferenceRecord { - String productname + String productvariant String subcomponentname String rawdifferencetext @@ -39,7 +39,7 @@ class DifferenceRecord { String comment DifferenceRecord() { - productname = "" + productvariant = "" subcomponentname = "" rawdifferencetext = "" @@ -56,7 +56,7 @@ class DifferenceRecord { for (j in jiraticketinfo) { jiraticketstring += j.dump() + " " } - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + return "Difference Record:\nproductvariant: ${productvariant}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -130,7 +130,7 @@ def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { } def String getcomponent(meta) { - def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefolder = "${meta.productvariant()}_${meta.productinfo.subcomponentname}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" @@ -343,7 +343,7 @@ def List fillrecord(meta, line, String sourcepath) { def List differencerecords = [] def DifferenceRecord dr = new DifferenceRecord() - dr.productname = meta.productinfo.productname + dr.productvariant = meta.productvariant() dr.subcomponentname = meta.productinfo.subcomponentname dr.rawdifferencetext = line @@ -755,7 +755,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def title(earlymeta, latermeta) { return """ Differences in: - Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} + Product: ${earlymeta.productvariant()} Subcomponent: ${earlymeta.productinfo.subcomponentname} between: then: Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} @@ -771,7 +771,7 @@ def compare(earlymetaitems, latermetaitems, key) { def earlymeta = earlymetaitems[key].metainfo def latermeta = latermetaitems[key].metainfo - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: Changed: ") def boolean repoavailable def String uniquefolderforsourcerepo @@ -817,14 +817,14 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) differencerecords = differencerecords + componentdifferencerecords } else { - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) summaryappend("\n") } } for (key in latermetaitems.keySet()) { if (!earlymetaitems.containsKey(key)) { - summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + summaryappend("${latermeta.productvariant()} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) summaryappend("\n") } diff --git a/documentation/doxygen_deploy.groovy b/documentation/doxygen_deploy.groovy index 5054c02..e2f74a7 100644 --- a/documentation/doxygen_deploy.groovy +++ b/documentation/doxygen_deploy.groovy @@ -30,10 +30,10 @@ pipeline { importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) - productname = doxygenmeta.productinfo.productname + productvariant = doxygenmeta.productvariant() buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix - echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}" + echo "Doyxgen imported for product: ${productvariant} and build stamp prefix: ${buildstampprefix}" } } } @@ -41,7 +41,7 @@ pipeline { steps { script { doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" - pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}" + pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productvariant}/${buildstampprefix}" echo "Copying from ${doxygensource} to ${pathonserver}" if (isUnix()) { sh """#!/bin/bash diff --git a/meta/folder.groovy b/meta/folder.groovy index ac3d0dd..34448dd 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -817,20 +817,20 @@ class MetaInfo extends GISJsonSerialised { String relativize(String mypath) { String pathRelative - script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + //script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") if (mypath.startsWith(script.WORKSPACE)) { pathRelative = mypath.minus(script.WORKSPACE) - script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + //script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") } else { pathRelative = mypath - script.echo("Path relative is unchanged: ${pathRelative}") + //script.echo("Path relative is unchanged: ${pathRelative}") } // Convert \ with / pathRelative = pathRelative.tokenize('\\').join('/') // Strip leading / if any pathRelative = pathRelative.replaceAll('^/+', '') - script.echo("Path relative is ${pathRelative}") + //script.echo("Path relative is ${pathRelative}") return pathRelative } @@ -845,7 +845,7 @@ class MetaInfo extends GISJsonSerialised { def String adjustedpath = relativize(destpath) def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { - script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") + //script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) @@ -1103,7 +1103,7 @@ class MetaInfo extends GISJsonSerialised { def ArrayList metacollection = new ArrayList() def Map metamap = new LinkedHashMap() def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) - script.echo("Release walker iteration info at start:\n${ii.show()}") + //script.echo("Release walker iteration info at start:\n${ii.show()}") def results = walkrelated(ii) return results.metacollection } diff --git a/product/info.groovy b/product/info.groovy index 3d7bc30..67d1449 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -28,7 +28,6 @@ class ProductDependency { def setreleasestate() { def LinkedHashMap results = rtscript.startswithspecialprefix(presplitreleasetag) - script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix releasestate = results.prefixfound @@ -94,6 +93,10 @@ class ProductParsed { scanstate = State.PARSED } + def markparseerror() { + scanstate = State.PARSED + } + def boolean parsed() { return (scanstate == State.PARSED) } @@ -103,20 +106,43 @@ class ProductParsed { } } -def addProductEntry(LinkedHashMap products, +def String translateproductvariant(productvarianttranslation, + String releaserepo, + String productvariant) { + def translatedproductvariant = productvarianttranslation.get(releaserepo) + if (translatedproductvariant != null) { + echo "Translating product variant: ${productvariant} to: ${translatedproductvariant}" + return translatedproductvariant + } + return productvariant +} + +def ProductParsed addProductEntry(LinkedHashMap productcollection, + productvarianttranslation, String productvariant, String releaserepo) { - def productparsed - if (products.get(releaserepo) == null) { // Use the first entry name only - productparsed = new ProductParsed(productvariant) - products.put(releaserepo, productparsed) + def ProductParsed productparsed + + if (productvariant && releaserepo) { + if (productcollection.get(releaserepo) == null) { // Use the first entry name only + echo "Adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" + def translatedproductvariant = translateproductvariant(productvarianttranslation, + releaserepo, + productvariant) + productparsed = new ProductParsed(translatedproductvariant) + productcollection.put(releaserepo, productparsed) + } else { + echo "Already added product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" + } + productparsed = productcollection.get(releaserepo) } else { - productparsed = products.get(releaserepo) + echo "Not adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" } return productparsed } -def parseproductdependencies(LinkedHashMap products, +def parseproductdependencies(LinkedHashMap productcollection, + productvarianttranslation, productreleasemeta, String productreleasefolder, String containingproductvariant, @@ -124,88 +150,148 @@ def parseproductdependencies(LinkedHashMap products, def dependentmeta = productreleasemeta.getassociated(productreleasefolder) for (meta in dependentmeta) { - def pd = new ProductDependency(this, - rt, - meta.productvariant(), - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - pd.setreleasestate() - if (meta.productinfo.subcomponentname == 'WHOLE') { - if (meta.productvariant() != containingproductvariant) { // Don't add myself - addProductEntry(products, - meta.productvariant(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) + def translatedproductvariant = translateproductvariant(productvarianttranslation, + meta.releaseinfo.releaserepoURL, + meta.productvariant()) + if (translatedproductvariant && meta.productinfo.subcomponentname) { + def pd = new ProductDependency(this, + rt, + translatedproductvariant, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + pd.setreleasestate() + if (meta.productinfo.subcomponentname == 'WHOLE' && translatedproductvariant) { + if (translatedproductvariant != containingproductvariant) { // Don't add myself + addProductEntry(productcollection, + productvarianttranslation, + translatedproductvariant, + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } + } else { + def pc = new ProductComponent(meta.productinfo.subcomponentname, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductComponent(pc) } - } else if (meta.productvariant() != containingproductvariant) { - addProductEntry(products, - meta.productvariant(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) } else { - def pc = new ProductComponent(meta.productinfo.subcomponentname, - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductComponent(pc) + echo "Ignoring unnamed dependency in ${productreleasemeta.productvariant()}" } } } -def parseproductrepo(LinkedHashMap products, +def parseproductrepo(LinkedHashMap productcollection, + productvarianttranslation, String productrepo, String productreleasefolder) { - def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) - def String productvariant - def ProductParsed pp + def boolean productrepogood = true + echo "Querying repo: ${productrepo}" + dir(productreleasefolder) { + deleteDir() + def returnstatus = sh(script: """#!/bin/bash + git init + if [ \$? -ne 0 ]; then exit 1; fi + git config remote.origin.url ${productrepo} + if [ \$? -ne 0 ]; then exit 1; fi + git config advice.detachedHead false + if [ \$? -ne 0 ]; then exit 1; fi + git sparse-checkout init + if [ \$? -ne 0 ]; then exit 1; fi + git sparse-checkout set meta/* + if [ \$? -ne 0 ]; then exit 1; fi + git pull -t + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true) + if (returnstatus > 0) { + dir('meta') { + deleteDir() + } + echo "Couldn't clone repository: ${productrepo} correctly" + productrepogood = false + } + } + if (productrepogood) { + def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) + def String productvariant + def ProductParsed pp + shatagmap.each { sha, tags -> + //echo "Sha: ${sha} has tags: ${tags}" + dir(productreleasefolder) { + def returnstatus = sh(script: """#!/bin/bash + git checkout ${sha} + """, + returnStatus: true) + if (returnstatus > 0) { + dir('meta') { + deleteDir() + } + } + } + def productreleasemeta = meta.importrelated(false, productreleasefolder) + if (!productvariant) { + productvariant = productreleasemeta.productvariant() + pp = addProductEntry(productcollection, + productvarianttranslation, + productvariant, + productrepo) + if (pp) { + echo "Product variant created for: ${productrepo} name: ${pp.productvariant}" + } else { + echo "No product entry added/associated with ${productrepo} name: ${productvariant}" + } + } - def ArrayList productreleases = new ArrayList() - shatagmap.each { sha, tags -> - def importoptions = importer.getNewImportOptions() - importoptions.releaserepo(productrepo, sha) - importoptions.setfilter('meta/**') - def productreleasemeta = importer.artifactimport(meta, - importoptions, - productreleasefolder) - - if (!productvariant) { - productvariant = productreleasemeta.productvariant() - pp = addProductEntry(products, - productvariant, productrepo) + if (pp) { + def productrelease = new ProductRelease(sha, + productreleasemeta.jenkinsbuildinfo.builddate, + tags) + echo "Productrelease: ${productrelease.sha} created" + parseproductdependencies(productcollection, + productvarianttranslation, + productreleasemeta, + productreleasefolder, + pp.productvariant, + productrelease) + echo "Productrelease: ${productrelease.sha} added to Productvariant: ${pp.productvariant}" + pp.addProductRelease(productrelease) + } + } + if (pp) { + pp.markparsed() + echo "Productvariant: ${pp.productvariant} parsed" } - - def productrelease = new ProductRelease(sha, - productreleasemeta.jenkinsbuildinfo.builddate, - tags) - parseproductdependencies(products, - productreleasemeta, - productreleasefolder, - productvariant, - productrelease) - pp.addProductRelease(productrelease) } - pp.markparsed() } def iterateinitialproductrepos(initialproductcollection, + productvarianttranslation, LinkedHashMap productcollection, String productreleasefolder) { for (productrepo in initialproductcollection.ProductReleaseRepos) { - echo "Querying ${productrepo}" parseproductrepo(productcollection, + productvarianttranslation, productrepo, productreleasefolder) } while( true ) { boolean allparsed = true + echo "Parsing extra products" productcollection.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false parseproductrepo(productcollection, + productvarianttranslation, releaserepo, productreleasefolder) + } else { + echo "Parsed already ${productparsed.productvariant}" } } - if (allparsed) break + if (allparsed) { + echo "All parsed" + break + } } } @@ -213,5 +299,4 @@ def LinkedHashMap getNewProductCollection() { return new LinkedHashMap() } - return this \ No newline at end of file diff --git a/publish/do.groovy b/publish/do.groovy index ea090f0..8e302d8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -22,7 +22,7 @@ def resetfolders() { } def productuniquefolder(releasemeta) { - return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + return "${releasemeta.productvariant()}_${releasemeta.productinfo.subcomponentname}" } def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { @@ -116,7 +116,7 @@ def recallcomponent(String productline, releaseinfo, String softwarerelease, ite def releasemeta = releaseinfo.metainfo def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + summaryappend("${releasemeta.productvariant()} ${releasemeta.productinfo.subcomponentname}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) @@ -145,7 +145,7 @@ def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, Str resetfolders() preconditions() - def productline = releasemeta.productinfo.productname + def productline = releasemeta.productvariant() // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) diff --git a/publish/package.groovy b/publish/package.groovy index 68c41f3..04236e0 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -8,30 +8,6 @@ pipeline { } } } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHONBRANCH = "origin/3.6.3_linux" - } - else { - PYTHONBRANCH = "origin/3.6.3_windows" - } - PYTHONDIR = "${WORKSPACE}/python" - PYTHONROOTDIR = 'python' - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) - if (isUnix()) { - env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" - } - else { - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - } - } - } stage('Clean exports') { steps { script { @@ -50,8 +26,16 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def packageimportoptions = importer.getNewImportOptions() - packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) - packageimportoptions.setincrementversionnumber(true) + // For backward compatibility - in case these parameters are not supplied + def packagelocation = params['PACKAGE_LOCATION'] ?: 'Jenkins job artifact' + def packagereleaserepo = params['PACKAGE_RELEASE_REPO'] ?: '' + def packagereleaserevision = params['PACKAGE_RELEASE_REVISION'] ?: '' + packageimportoptions.source(packagelocation, + PACKAGE_CREATE_JOB, + PACKAGE_CREATE_JOB_SELECTOR, + packagereleaserepo, + packagereleaserevision) + packageimportoptions.setincrementversionnumber(false) packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) PACKAGE_FOLDER = "${PACKAGEDEST}/package" diff --git a/publish/packagecollection.groovy b/publish/packagecollection.groovy new file mode 100644 index 0000000..a87453e --- /dev/null +++ b/publish/packagecollection.groovy @@ -0,0 +1,183 @@ +def packagecollection = null + +/* Example expected JSON +{ +{ + "packages": [ + { + "Name": "gis-hmb-fd-hv-esw-el-sg1024", + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_app_0.9.6_debpackage", + "Job selector": "1248", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_package.git", + "Release repo revision": "refs/tag/0.9.6.57669", + "Path": "package/gis-hmb-fd-hv-esw-el-sg1024_armhf.deb" + }, + { + "Name": "gis-hmb-fd-hv-control-service-sg1024", + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_control_0.9.6_debpackage", + "Job selector": "1722", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_control_package.git", + "Release repo revision": "refs/tag/0.9.6.7964", + "Path": "package/gis-hmb-fd-hv-control-service-sg1024_armhf.deb" + } + ] +} +*/ + +def String asJsonText(somejson = packagecollection) { + return writeJSON(returnText: true, json: somejson, pretty: 1) +} + +def boolean packagecollection_exists() { + return (packagecollection != null) +} + +def packagecollection_set(String jsontext) { + packagecollection = readJSON(text: jsontext) + echo "Package collection info:\n${asJsonText(packagecollection)}\nType:\n${packagecollection.getClass()}" +} + +def packagecollection_get() { + return packagecollection +} + +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + PACKAGEDEST = "${WORKSPACE}/package" + cleanfolders = [ + PACKAGEDEST, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Transfer packages') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + packagecollection_set(PACKAGE_COLLECTION) + debpackagecollection = packagecollection_get() + + debpackagecollection.packages.each { debpackageinfo-> + echo "Processing package: ${debpackageinfo.Name}\n${debpackageinfo}\n${debpackageinfo.getClass()}" + echo "Info: ${debpackageinfo.Location}" + echo "Info: ${debpackageinfo['Location']}" + echo "Info: ${debpackageinfo['Job']}" + echo "Info: ${debpackageinfo['Job selector']}" + echo "Info: ${debpackageinfo['Release repo']}" + echo "Info: ${debpackageinfo['Release repo revision']}" + + def packagedest = "${PACKAGEDEST}/${debpackageinfo.Name}" + + def packageimportoptions = importer.getNewImportOptions() + // For backward compatibility - in case these parameters are not supplied + packageimportoptions.source(debpackageinfo['Location'], + debpackageinfo['Job'], + "${debpackageinfo['Job selector']}", + debpackageinfo['Release repo'], + debpackageinfo['Release repo revision']) + packageimportoptions.setincrementversionnumber(false) + packagemeta = importer.artifactimport(meta, packageimportoptions, packagedest) + + UNIQUE_PACKAGE_FOLDER = "${packagedest}/package" + + echo "Aptly config file is ${APTLY_CONFIG_FILE}" + echo "Repo name is ${REPO_NAME}" + echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" + + sh """#!/bin/bash -v + aptly config show + if [ \$? -ne 0 ]; then exit 1; fi + + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then + echo "aptly repo ${REPO_NAME} does not exist; Creating it" + aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + fi + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" + // Convert prefix to regular expression + PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") + PACKAGE_VERSION_REGEX = '' + i = 0 + while(true) { + if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + break; + } + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + PACKAGE_VERSION_REGEX += '\\.' + i++ + } + sh """#!/bin/bash -v + # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' + if [ \$? -ne 0 ]; then exit 1; fi + aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/do.groovy b/release/do.groovy index 4dcb091..05de91f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -191,7 +191,7 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}" taglist.add(componentothertagitem) } } @@ -377,13 +377,13 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { def buildmeta = buildinfo.metainfo def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) - def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" + def String uniquefolder = "${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") - repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + summaryappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) diff --git a/somsoftware/sominfo.groovy b/somsoftware/sominfo.groovy new file mode 100644 index 0000000..a7fdc82 --- /dev/null +++ b/somsoftware/sominfo.groovy @@ -0,0 +1,31 @@ +def somsoftwareinfostore = null + +/* Example expected JSON +{ + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "SOM_GIS_master/SOM_GIS_master_build", + "Job selector": "true", + "Release repo": "git@hydra:som_gis-releases/som_gis_whole.git", + "Release repo revision": "" + } +}*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: somsoftwareinfostore, pretty: 1) +} + +def boolean exists() { + return (somsoftwareinfostore != null) +} + +def set(String jsontext) { + somsoftwareinfostore = readJSON(text: jsontext) + echo "Som software support info:\n${asJsonText()}" +} + +def get() { + return somsoftwareinfostore +} + +return this \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 61378b5..2b6fb09 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -43,7 +43,6 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL def shatagfields = shatag.split('\\t') def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() - echo "sha: ${sha} tag: ${tag}" if (shatagmap.get(sha) == null) { def ArrayList taglist = new ArrayList() taglist.add(tag) @@ -54,6 +53,9 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL shatagmap.put(sha, taglist) } } + shatagmap.each { sha, tags -> + echo "sha: ${sha} has tags: ${tags}" + } return shatagmap } From 19d5febca0d02b69a39d4b90d65179b0866cb9bb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 23 Mar 2022 08:24:21 +0000 Subject: [PATCH 70/73] Adopt the build stamp of the contained product --- install/install.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install/install.groovy b/install/install.groovy index d5b495e..257b35c 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -158,6 +158,8 @@ pipeline { controlpackagemeta = importer.artifactimport(meta, controlpackageimportoptions, CONTROLPACKAGEIMPORTPATH) BUILD_STAMP = apppackagemeta.releasetag() + version.pregenerated(meta, SOURCE_REVISION, SOURCE_PATH, BUILD_STAMP) // Adopt the version of the contained product + PRODUCT_NAME = apppackagemeta.productinfo.productname EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${RELEASE_REPO_VARIANT_SUFFIX} ${BUILD_STAMP} LNP Updater" From 268b7de902dc7583fc144f99780bfc95d58889e3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 23 Mar 2022 10:51:08 +0000 Subject: [PATCH 71/73] Adopt the build stamp of the contained product --- install/install.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/install.groovy b/install/install.groovy index 257b35c..f37eeed 100644 --- a/install/install.groovy +++ b/install/install.groovy @@ -157,9 +157,8 @@ pipeline { controlpackageimportoptions.setincrementversionnumber(false) controlpackagemeta = importer.artifactimport(meta, controlpackageimportoptions, CONTROLPACKAGEIMPORTPATH) + version.pregenerated(meta, SOURCE_REVISION, SOURCE_PATH, apppackagemeta.buildstampinfo.buildstamp) // Adopt the version of the contained product BUILD_STAMP = apppackagemeta.releasetag() - version.pregenerated(meta, SOURCE_REVISION, SOURCE_PATH, BUILD_STAMP) // Adopt the version of the contained product - PRODUCT_NAME = apppackagemeta.productinfo.productname EXE_NAME_PREFIX = "GIS ${PRODUCT_NAME} ${RELEASE_REPO_VARIANT_SUFFIX} ${BUILD_STAMP} LNP Updater" From 9489dbd59500fb5d24b88a9d33fe64c84e73b97a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 23 Mar 2022 20:40:54 +0000 Subject: [PATCH 72/73] Squashed 'Common_scripts/' changes from 791c94c..98bc43b 98bc43b Use the autobuild script revision count to alter the version 659d0a0 Use the autobuild script revision count to alter the version 727be2d Use the autobuild script revision count to alter the version 554e551 Use the autobuild script revision count to alter the version a517c02 Use the autobuild script revision count to alter the version git-subtree-dir: Common_scripts git-subtree-split: 98bc43bd9ea1b5adeebd4ad84bde71ac371b9a62 --- Common_scripts/versionnumbering/generate.groovy | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Common_scripts/versionnumbering/generate.groovy b/Common_scripts/versionnumbering/generate.groovy index 5e974e2..40413d7 100644 --- a/Common_scripts/versionnumbering/generate.groovy +++ b/Common_scripts/versionnumbering/generate.groovy @@ -11,7 +11,7 @@ int incrementrevision(int current, int increment) { return newrevision } -def getGitRevision(String gitrootpath, int basebuildnumber) { +def getGitRevision(String gitrootpath, int basebuildnumber = 0) { def int adjustedchangecount def String gitrevision def String remoteinfo @@ -143,16 +143,23 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum def String gitrevision def String remoteinfo def String remoteurl - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) + def int scriptgitrevisioncount + def String scriptgitrevision + def String scriptremoteinfo + def String scriptremoteurl + (scriptgitrevisioncount, scriptgitrevision, scriptremoteinfo, scriptremoteurl) = getGitRevision(autobuilder.getautobuildpath()) + + def int totalrevisioncount = incrementrevision(gitrevisioncount, scriptgitrevisioncount) + def versionprefix = getVersionPrefix(sourcerevision) - def buildstamp = versionprefix + gitrevisioncount.toString() + def buildstamp = versionprefix + totalrevisioncount.toString() echo "Build stamp is: ${buildstamp}" meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) + meta.buildstampinfo.set(versionprefix, totalrevisioncount, buildstamp) return buildstamp } From add1a353a77370ae0f50540bf5302f4465cb3be1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 23 Mar 2022 20:54:12 +0000 Subject: [PATCH 73/73] Remove broken subtree --- Common_scripts/.project | 17 - Common_scripts/.pydevproject | 8 - .../org.eclipse.core.resources.prefs | 2 - Common_scripts/artifacts/archiver.groovy | 17 - Common_scripts/artifacts/metaexport.groovy | 26 - Common_scripts/autobuilder.groovy | 147 -- Common_scripts/build/integrity.groovy | 113 -- Common_scripts/build/postactions.groovy | 229 ---- Common_scripts/build/postactionstest.groovy | 43 - Common_scripts/check/build_meta.groovy | 87 -- Common_scripts/check/do_meta.groovy | 77 -- Common_scripts/checksum/utils.groovy | 15 - Common_scripts/clean/clean.groovy | 10 - Common_scripts/commandline/commandline.groovy | 12 - .../eclipsebuild/disableparallelbuilds.groovy | 12 - .../eclipsebuild/eclipseSTM8build.groovy | 172 --- .../eclipsebuild/eclipseds5build.groovy | 252 ---- .../eclipsebuild/eclipseniosbuild.groovy | 116 -- .../configuration/configuration.groovy | 18 - Common_scripts/configuration/tools.groovy | 59 - Common_scripts/copy/all.groovy | 67 - Common_scripts/copy/esw.groovy | 67 - Common_scripts/copy/filtered.groovy | 73 - Common_scripts/copy/logic.groovy | 79 -- Common_scripts/differences/.project | 17 - Common_scripts/differences/.pydevproject | 8 - .../differences/dometahierarchy.groovy | 836 ------------ Common_scripts/differences/file.groovy | 24 - Common_scripts/differences/json.groovy | 51 - .../differences/metahierarchy.groovy | 102 -- .../documentation/doxygen_deploy.groovy | 80 -- Common_scripts/hardware/sominfo.groovy | 80 -- Common_scripts/import/importer.groovy | 408 ------ Common_scripts/install/installhelper.groovy | 136 -- Common_scripts/meta/folder.groovy | 1201 ----------------- Common_scripts/packaging/debian.groovy | 111 -- Common_scripts/platform/lineendings.groovy | 5 - Common_scripts/product/info.groovy | 302 ----- Common_scripts/publish/.classpath | 9 - Common_scripts/publish/.gitignore | 7 - Common_scripts/publish/.project | 18 - .../org.eclipse.jdt.groovy.core.prefs | 3 - Common_scripts/publish/do.groovy | 174 --- Common_scripts/publish/package.groovy | 165 --- .../publish/packagecollection.groovy | 183 --- .../publish/promptingpublish.groovy | 184 --- Common_scripts/publish/publish.groovy | 92 -- Common_scripts/publish/publishitems.groovy | 129 -- Common_scripts/publishing/utils.groovy | 27 - Common_scripts/release/do.groovy | 458 ------- Common_scripts/release/registerrepo.groovy | 12 - Common_scripts/release/release.groovy | 87 -- Common_scripts/release/tag.groovy | 39 - Common_scripts/remoteaccess/ftp.groovy | 32 - Common_scripts/remoteaccess/hostedftp.groovy | 28 - Common_scripts/somsoftware/sominfo.groovy | 31 - Common_scripts/staticanalysis/ccm.groovy | 85 -- Common_scripts/staticanalysis/cpd.groovy | 152 --- Common_scripts/staticanalysis/cppcheck.groovy | 114 -- .../test/testethernetproduct.groovy | 204 --- Common_scripts/textutils/subst.groovy | 8 - .../vs2017install/buildmachines.vsconfig | 25 - Common_scripts/versioncontrol/git.groovy | 402 ------ Common_scripts/versionnumbering/do.groovy | 23 - .../versionnumbering/generate.groovy | 187 --- Common_scripts/versionstamping/.project | 17 - Common_scripts/versionstamping/.pydevproject | 8 - .../versionstamping/versionstamper.py | 38 - 68 files changed, 8020 deletions(-) delete mode 100644 Common_scripts/.project delete mode 100644 Common_scripts/.pydevproject delete mode 100644 Common_scripts/.settings/org.eclipse.core.resources.prefs delete mode 100644 Common_scripts/artifacts/archiver.groovy delete mode 100644 Common_scripts/artifacts/metaexport.groovy delete mode 100644 Common_scripts/autobuilder.groovy delete mode 100644 Common_scripts/build/integrity.groovy delete mode 100644 Common_scripts/build/postactions.groovy delete mode 100644 Common_scripts/build/postactionstest.groovy delete mode 100644 Common_scripts/check/build_meta.groovy delete mode 100644 Common_scripts/check/do_meta.groovy delete mode 100644 Common_scripts/checksum/utils.groovy delete mode 100644 Common_scripts/clean/clean.groovy delete mode 100644 Common_scripts/commandline/commandline.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/eclipseSTM8build.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/eclipseds5build.groovy delete mode 100644 Common_scripts/compile/eclipsebuild/eclipseniosbuild.groovy delete mode 100644 Common_scripts/configuration/configuration.groovy delete mode 100644 Common_scripts/configuration/tools.groovy delete mode 100644 Common_scripts/copy/all.groovy delete mode 100644 Common_scripts/copy/esw.groovy delete mode 100644 Common_scripts/copy/filtered.groovy delete mode 100644 Common_scripts/copy/logic.groovy delete mode 100644 Common_scripts/differences/.project delete mode 100644 Common_scripts/differences/.pydevproject delete mode 100644 Common_scripts/differences/dometahierarchy.groovy delete mode 100644 Common_scripts/differences/file.groovy delete mode 100644 Common_scripts/differences/json.groovy delete mode 100644 Common_scripts/differences/metahierarchy.groovy delete mode 100644 Common_scripts/documentation/doxygen_deploy.groovy delete mode 100644 Common_scripts/hardware/sominfo.groovy delete mode 100644 Common_scripts/import/importer.groovy delete mode 100644 Common_scripts/install/installhelper.groovy delete mode 100644 Common_scripts/meta/folder.groovy delete mode 100644 Common_scripts/packaging/debian.groovy delete mode 100644 Common_scripts/platform/lineendings.groovy delete mode 100644 Common_scripts/product/info.groovy delete mode 100644 Common_scripts/publish/.classpath delete mode 100644 Common_scripts/publish/.gitignore delete mode 100644 Common_scripts/publish/.project delete mode 100644 Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs delete mode 100644 Common_scripts/publish/do.groovy delete mode 100644 Common_scripts/publish/package.groovy delete mode 100644 Common_scripts/publish/packagecollection.groovy delete mode 100644 Common_scripts/publish/promptingpublish.groovy delete mode 100644 Common_scripts/publish/publish.groovy delete mode 100644 Common_scripts/publish/publishitems.groovy delete mode 100644 Common_scripts/publishing/utils.groovy delete mode 100644 Common_scripts/release/do.groovy delete mode 100644 Common_scripts/release/registerrepo.groovy delete mode 100644 Common_scripts/release/release.groovy delete mode 100644 Common_scripts/release/tag.groovy delete mode 100644 Common_scripts/remoteaccess/ftp.groovy delete mode 100644 Common_scripts/remoteaccess/hostedftp.groovy delete mode 100644 Common_scripts/somsoftware/sominfo.groovy delete mode 100644 Common_scripts/staticanalysis/ccm.groovy delete mode 100644 Common_scripts/staticanalysis/cpd.groovy delete mode 100644 Common_scripts/staticanalysis/cppcheck.groovy delete mode 100644 Common_scripts/test/testethernetproduct.groovy delete mode 100644 Common_scripts/textutils/subst.groovy delete mode 100644 Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig delete mode 100644 Common_scripts/versioncontrol/git.groovy delete mode 100644 Common_scripts/versionnumbering/do.groovy delete mode 100644 Common_scripts/versionnumbering/generate.groovy delete mode 100644 Common_scripts/versionstamping/.project delete mode 100644 Common_scripts/versionstamping/.pydevproject delete mode 100644 Common_scripts/versionstamping/versionstamper.py diff --git a/Common_scripts/.project b/Common_scripts/.project deleted file mode 100644 index e329c3f..0000000 --- a/Common_scripts/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - Common Build Scripts - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/Common_scripts/.pydevproject b/Common_scripts/.pydevproject deleted file mode 100644 index 1ce5929..0000000 --- a/Common_scripts/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - - -/${PROJECT_DIR_NAME} - -python 2.7 -python35 - diff --git a/Common_scripts/.settings/org.eclipse.core.resources.prefs b/Common_scripts/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 1174a06..0000000 --- a/Common_scripts/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,2 +0,0 @@ -eclipse.preferences.version=1 -encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/Common_scripts/artifacts/archiver.groovy b/Common_scripts/artifacts/archiver.groovy deleted file mode 100644 index bafba02..0000000 --- a/Common_scripts/artifacts/archiver.groovy +++ /dev/null @@ -1,17 +0,0 @@ -enum Store { - JENKINS, - ARTIFACTSERVER, -} - -this.JENKINS = Store.JENKINS -this.ARTIFACTSERVER = Store.JENKINS - -def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { - echo "Archiving artifacts to store: ${store}" - meta.copyrelatedmeta() - if (store == JENKINS) { - archiveArtifacts(artifacts: artifactlist) - } -} - -return this \ No newline at end of file diff --git a/Common_scripts/artifacts/metaexport.groovy b/Common_scripts/artifacts/metaexport.groovy deleted file mode 100644 index a40e9a8..0000000 --- a/Common_scripts/artifacts/metaexport.groovy +++ /dev/null @@ -1,26 +0,0 @@ -// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(meta, importer, subJobs) { - def ArrayList artifactFolders = new ArrayList() - for (subJob in subJobs) { - def subJobObject = subJob[0] - if (subJobObject != null) { - def subJobName = subJobObject.getFullProjectName() - def subJobNumber = subJobObject.getNumber() - def destFolder = subJob[1] - def boolean incrementversionnumber = subJob[2] - dir(destFolder) { - deleteDir() - } - - def importoptions = importer.getNewImportOptions() - importoptions.jenkinsjob(subJobName, subJobNumber) - importoptions.setincrementversionnumber(incrementversionnumber) - importer.artifactimport(meta, importoptions, destFolder) - - artifactFolders.add("${destFolder}/**") - } - } - return artifactFolders -} - -return this; diff --git a/Common_scripts/autobuilder.groovy b/Common_scripts/autobuilder.groovy deleted file mode 100644 index 667f89a..0000000 --- a/Common_scripts/autobuilder.groovy +++ /dev/null @@ -1,147 +0,0 @@ -// Assume that WORKSPACE environment variable is set - -autobuildroot = 'autobuild' -commonscriptsroot = "${autobuildroot}/Common_scripts" -metafolderscript = null - -def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { - if (doclean) { - echo "Cleaning workspace" - // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - def stashname = 'Clean workspace stash' - stash(allowEmpty: true, - includes: "${autobuildroot}/**,${autobuildroot}/.git/**", - name: stashname, - useDefaultExcludes: false - ) - cleanWs() - unstash name: stashname - } - else { - echo "Not cleaning workspace" - } -} - -def commonscripts() { - return commonscriptsroot -} - -def getautobuildpath() { - return autobuildroot -} - -def getmetafolderscript() { - return metafolderscript -} - -def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { - def String altered = releaserepo.trim() - - if (releaserepovariantsuffix?.trim()) { - altered = altered.replace("-RELEASE_REPO_VARIANT_SUFFIX-", releaserepovariantsuffix) - } - return altered -} - -def setBuildType(meta, boolean dofullbuild = true) { - def doquickbuild = false - - echo "Build trigger cause: ${currentBuild.getBuildCauses()}" - if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { - echo "Build triggered by user" - if (!dofullbuild) { - echo "Marking as quick build as user wanted one" - doquickbuild = true - } - } else { - echo "Build triggered automatically" - if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { - echo "Marking as quick build as it was triggered by a commit" - doquickbuild = true - } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { - echo "Marking as full build as it was triggered by timer" - } - } - echo "Build type is ${doquickbuild ? "quick" : "full"}" - if (doquickbuild) { - unstable('This not a full build - a subset of the artifacts is not being generated') - } - meta.jenkinsbuildinfo.set(!doquickbuild) - return doquickbuild -} - -def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { - def meta = getmetafolderscript().metareset(script) - echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" - meta.productinfo.set(productname, subcomponentname, productvariant) - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - for (p in params) { - meta.jenkinsbuildinfo.addParameter(p.key, p.value) - } - return meta -} - -// Start up boilerplate -def startbuild( - script, - String productname = PRODUCT_NAME, - String subcomponentname = SUBCOMPONENT_NAME, - String releaserepo = RELEASE_REPO, - String productvariant = '', - String autobuildpath = autobuildroot, - String commonscriptspath = commonscriptsroot - ) { - echo "Starting...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - metafolderscript = load("${commonscripts()}/meta/folder.groovy") - - conditionalclean() - - def meta = setupmeta(script, productname, subcomponentname, productvariant) - - def rr = load("${commonscripts()}/release/registerrepo.groovy") - def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) - - echo "Release repo: ${alteredreleaserepo}" - rr.registerrepo(meta, alteredreleaserepo) - - def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration(meta) - - return meta -} - -// Start up boilerplate -def startnonbuild( - script, - autobuildpath = autobuildroot, - commonscriptspath = commonscriptsroot - ) { - echo "Starting non-build job...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - metafolderscript = load("${commonscripts()}/meta/folder.groovy") - - conditionalclean() - - def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") - - def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration(meta) - - return meta -} - -def configurationhelper() { - return load("${commonscripts()}/configuration/configuration.groovy") -} - -// Everyone wants to know this -autobuilder = this - -return this; diff --git a/Common_scripts/build/integrity.groovy b/Common_scripts/build/integrity.groovy deleted file mode 100644 index e93c00f..0000000 --- a/Common_scripts/build/integrity.groovy +++ /dev/null @@ -1,113 +0,0 @@ -import static groovy.json.JsonOutput.* - -def check(buildmeta) { - def results = [ - 'good': true, - 'comment': '', - ] - - // Walk the meta looking for different versions of the same components - def allmeta = buildmeta.allrelatedmeta() - echo "All meta:\n${allmeta.comment}" - - // Store build stamps associated with each product component; Store a counter against each product component version - def productcomponents = [:] - for (meta in allmeta.metacollection) { - def key = meta.componentidentifier() - - if (!productcomponents.containsKey(key)) { - productcomponents[key] = [:] - } - def buildstampnumber = meta.releasetag() - if (!productcomponents[key].containsKey(buildstampnumber)) { - productcomponents[key][buildstampnumber] = 1 - } else { - productcomponents[key][buildstampnumber] += 1 - } - } - - // Show what was collected - echo prettyPrint(toJson(productcomponents)) - - // Look for product components with more than one build stamp - for (pc in productcomponents.keySet()) { - def bsns = productcomponents[pc] - if (bsns.size() > 1) { - results.good = false - results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" - for (bsn in bsns.keySet()) { - results.comment += "${bsn}\n" - } - } - } - - // Store product components associated with each release git url - def releasegiturls = [:] - for (meta in allmeta.metacollection) { - def String key = meta.releaseinfo.releaserepoURL - - if (key) { - if (!releasegiturls.containsKey(key)) { - releasegiturls[key] = [:] - } - def componentidentifier = meta.componentidentifier() - if (!releasegiturls[key].containsKey(componentidentifier)) { - releasegiturls[key][componentidentifier] = 1 - } else { - releasegiturls[key][componentidentifier] += 1 - } - } - } - - // Show what was collected - echo prettyPrint(toJson(releasegiturls)) - - // Look for release git urls being used by more than one product component - for (rgu in releasegiturls.keySet()) { - def cis = releasegiturls[rgu] - if (cis.size() > 1) { - results.good = false - results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" - for (ci in cis.keySet()) { - results.comment += "${ci}\n" - } - } - } - - // Store git commit shas associated with each source git url - def sourcegiturls = [:] - for (meta in allmeta.metacollection) { - def String key = meta.gitinfo.remoteurl - - if (key) { - if (!sourcegiturls.containsKey(key)) { - sourcegiturls[key] = [:] - } - def commitsha = meta.gitinfo.gitrevision - if (!sourcegiturls[key].containsKey(commitsha)) { - sourcegiturls[key][commitsha] = 1 - } else { - sourcegiturls[key][commitsha] += 1 - } - } - } - - // Show what was collected - echo prettyPrint(toJson(sourcegiturls)) - - // Look for source git urls associated with more than one sha code - for (sgu in sourcegiturls.keySet()) { - def cs = sourcegiturls[sgu] - if (cs.size() > 1) { - results.good = false - results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" - for (c in cs.keySet()) { - results.comment += "${c}\n" - } - } - } - - return results -} - -return this \ No newline at end of file diff --git a/Common_scripts/build/postactions.groovy b/Common_scripts/build/postactions.groovy deleted file mode 100644 index 3206004..0000000 --- a/Common_scripts/build/postactions.groovy +++ /dev/null @@ -1,229 +0,0 @@ -/* Example post build info -{ - "Build outcomes": { - "Success": { - "Actions": { - "Deploy documentation": { - "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", - "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" - }, - "Static analysis": { - "Description": "Can be empty - Static analysis job to run when this build is successful", - "Job": "" - }, - "Unit test": { - "Description": "Can be empty - Unit test job to run when this build is successful", - "Job": "" - }, - "Regression test": { - "Description": "Can be empty - Regression test job to run when this build is successful", - "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" - }, - "Update differences": { - "Description": "Can be empty - Update differences job to run when this build is successful", - "Job": "HMB_FD_HV_diff_releases" - }, - "Release to git": { - "Description": "Can be empty - Name of release to git job to run if this build is successful", - "Job": "HMB_FD_HV_release_to_git" - } - } - }, - "Failed": { - "Actions": { - "Email": { - "Description": "Can be empty - Name of release to git job to run if this build is successful", - "To": [ - "dan.hawkins@globalinkjetsystems.com" - ] - }, - "ExtEmail": { - "Description": "Send an email using Extended Email plugin", - "To": [ - "dan.hawkins@globalinkjetsystems.com" - ], - "Recipient providers": [ - "culprits", - "requestor", - "developers", - "upstreamDevelopers" - ] - } - } - } - } -} -*/ - -def getActionJobName(postbuildactions, String action, String alternativeparamname) { - def String jobname = '' - if (postbuildactions) { - try { - jobname = postbuildactions[action]['Job'] - } catch(Exception ex) { - echo "No post build actions found for action: ${action}" - jobname = '' - } - } else { - if (params[alternativeparamname] != null) { - if (params[alternativeparamname]) { - jobname = params[alternativeparamname] - } - } - } - if (jobname) { - def jobitem = Jenkins.instance.getItemByFullName(jobname) - if (jobitem) { - if (!jobitem.isBuildable()) { - jobname = '' - } - } else { - jobname = '' - } - } - // echo "Job name for action: ${action}: ${jobname}" - return jobname -} - -def deployjob(meta, String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername, String releaserepoparametername = '') { - def jobname = getActionJobName(postbuildactions, action, alternativeparamname) - if (jobname) { - def parameters = [ - new StringParameterValue(jobnameparametername, JOB_NAME), - new StringParameterValue(jobselectorparametername, buildselector), - ] - if (releaserepoparametername) { - parameters += new StringParameterValue(releaserepoparametername, meta.releaseinfo.releaserepoURL) - } - build( - wait: false, - job: jobname, - parameters: parameters - ) - } -} - -def readPostBuildInfo() { - def postbuildinfo = readJSON(text: POST_BUILD_INFO) - def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) - // echo "Post build info:\n${postbuildjsontext}" - - return postbuildinfo -} - -def performSuccessfulBuildActions(meta) { - def postbuildactions - if (params.POST_BUILD_INFO != null) { - def postbuildinfo = readPostBuildInfo() - try { - postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] - } catch(Exception ex) { - echo "Post build info did not contain: Build outcomes/Success/Actions" - postbuildactions = null - } - } - def String buildselector = "${BUILD_NUMBER}" - deployjob(meta, buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') - deployjob(meta, buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) - deployjob(meta, buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(meta, buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(meta, buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR', 'TOP_LEVEL_META_REPOSITORY') - deployjob(meta, buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') -} - -def sendemails(emailinfo) { - def emailtolist = emailinfo['To'] - - if (emailtolist) { - def emailbody = """ - Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} - - Console output: ${BUILD_URL}console - """ - for(emailto in emailtolist) { - mail( - subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", - body: emailbody, - to: emailto - ) - // echo "Send email to: ${emailto}" - } - } -} - -def getRecipientProviders(recipientProviderInfoList) { - def recipientProviderMap = [ - culprits: [$class: 'CulpritsRecipientProvider'], - requestor: [$class: 'RequesterRecipientProvider'], - developers: [$class: 'DevelopersRecipientProvider'], - upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], - ] - def recipientProviders = [] - - if (recipientProviderInfoList) { - for(rp in recipientProviderInfoList) { - if (recipientProviderMap.containsKey(rp)) { - recipientProviders.add(recipientProviderMap[rp]) - } - } - } - if (recipientProviders.size() == 0) { - recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] - } - // echo "recipientProviders: ${recipientProviders.toString()}" - return recipientProviders; -} - -def String getToList(toListInfo) { - def String toList = '' - - if (toListInfo) { - toList = toListInfo.join(',') - } - // echo "toList: ${toList}" - return toList -} - -def sendextemail(extmailinfo) { - def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) - def toList = getToList(extmailinfo['To']) - - emailext( - subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", - body: '''${JELLY_SCRIPT,template="html"}''', - attachLog: true, - // attachmentsPattern: '', - compressLog: false, - // from : '', - mimeType: 'text/html', - // postsendScript: '', - // presendScript: '', - recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers - // replyTo: '', - saveOutput: false, - to: toList, - ) -} - -def performFailedBuildActions(meta) { - def postbuildactions = null - if (params.POST_BUILD_INFO != null) { - def postbuildinfo = readPostBuildInfo() - try { - postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] - } catch(Exception ex) { - echo "Post build info did not contain: Build outcomes/Failed/Actions" - postbuildactions = null - } - } - if (postbuildactions) { - if (postbuildactions['Email']) { - sendemails(postbuildactions['Email']) - } - if (postbuildactions['ExtEmail']) { - sendextemail(postbuildactions['ExtEmail']) - } - } -} - -return this; diff --git a/Common_scripts/build/postactionstest.groovy b/Common_scripts/build/postactionstest.groovy deleted file mode 100644 index 150f5d1..0000000 --- a/Common_scripts/build/postactionstest.groovy +++ /dev/null @@ -1,43 +0,0 @@ -pipeline { - agent { label 'master' } - stages { - stage('Initial setup') { - steps { - script { - CLEAN_WORKSPACE = false - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Dummy build job') { - steps { - script { - echo "Execute dummy build job: ${CAUSE_ERROR}" - if (CAUSE_ERROR == 'true') { - error("Causing error") - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - script { - load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() - } - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - script { - load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() - } - } - } -} diff --git a/Common_scripts/check/build_meta.groovy b/Common_scripts/check/build_meta.groovy deleted file mode 100644 index fb8a75d..0000000 --- a/Common_scripts/check/build_meta.groovy +++ /dev/null @@ -1,87 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - cleanfolders = [ - "${WORKSPACE}/build_artifacts", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Get build artifact meta') { - steps { - script { - BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" - copyArtifacts( - projectName: BUILD_JOB, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR - ], - target: BUILD_ARTIFACTS - ) - def mfs = autobuilderscript.getmetafolderscript() - buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) - } - } - } - stage('Check components') { - steps { - script { - def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") - (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/check/do_meta.groovy b/Common_scripts/check/do_meta.groovy deleted file mode 100644 index 4b8a89e..0000000 --- a/Common_scripts/check/do_meta.groovy +++ /dev/null @@ -1,77 +0,0 @@ -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" -releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" -tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Release summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason += reason + "\n" - echo "Marked build as unstable: Reason: ${reason}" -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { - int md = maxdepth.toInteger() - resetfolders() - preconditions() - - // Walk the meta information to a max depth - def buildinfo = buildmeta.releasewalker(md) - echo "All components:" - def String componentinfo = "" - for (entry in buildinfo) { - def name = entry.metainfo.componentversionuniqueidentifier() - def offsetpath = entry.accumulatedpath - def abspath = "${buildpathroot}/${offsetpath}" - componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" - } - echo componentinfo - - // Walk the meta information to a max depth - def componentReleaseInfo = buildmeta.releasewalkerunique(md) - echo "Unique components:" - def String versioncomponent = "" - for (key in componentReleaseInfo.keySet()) { - def buildinfo2 = componentReleaseInfo[key] - def offsetpath = buildinfo2.accumulatedpath - def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" - } - echo versioncomponent - - return [summarystring, markunstable, unstablereason] -} - -return this; diff --git a/Common_scripts/checksum/utils.groovy b/Common_scripts/checksum/utils.groovy deleted file mode 100644 index be7569d..0000000 --- a/Common_scripts/checksum/utils.groovy +++ /dev/null @@ -1,15 +0,0 @@ - -// Generate a checksum file of from a list of files/directories relative to workspace -def generatelist(meta, checksummable) { - for (filepath in checksummable) { - def checksumset = meta.checksuminfo.addChecksumSet(filepath) - def files = findFiles(glob: filepath) - for (file in files) { - echo "Checksumming: ${file.path}" - def sum = sha1(file: file.path).trim() - checksumset.addChecksumPair(file.path, sum) - } - } -} - -return this; \ No newline at end of file diff --git a/Common_scripts/clean/clean.groovy b/Common_scripts/clean/clean.groovy deleted file mode 100644 index 4707ba0..0000000 --- a/Common_scripts/clean/clean.groovy +++ /dev/null @@ -1,10 +0,0 @@ - -def clean(folderlist) { - for (folder in folderlist) { - dir(folder) { - deleteDir() - } - } -} - -return this; \ No newline at end of file diff --git a/Common_scripts/commandline/commandline.groovy b/Common_scripts/commandline/commandline.groovy deleted file mode 100644 index 38f80fa..0000000 --- a/Common_scripts/commandline/commandline.groovy +++ /dev/null @@ -1,12 +0,0 @@ -String getCommandOutput(String cmd) { - if (isUnix()) { - return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() - } else { - return powershell(returnStdout: true, script: """ - ${cmd} - """ - ).trim() - } -} - -return this \ No newline at end of file diff --git a/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy b/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy deleted file mode 100644 index 03e4c8f..0000000 --- a/Common_scripts/compile/eclipsebuild/disableparallelbuilds.groovy +++ /dev/null @@ -1,12 +0,0 @@ - -// Disable parallel make builds - which may be causing autobuild issues -def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { - if (isUnix()) { - sh """ - # Looks for " sparseitemlist = gh.convertSparseItemsString(filter) - gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - - return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) -} - -return this diff --git a/Common_scripts/configuration/tools.groovy b/Common_scripts/configuration/tools.groovy deleted file mode 100644 index dbaeedb..0000000 --- a/Common_scripts/configuration/tools.groovy +++ /dev/null @@ -1,59 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -class ToolOptions { - def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - - String m_toolName - - String m_gitURL - String m_revision - String m_repoRoot - int m_cloneDepth - boolean m_shallowClone - - ToolOptions(script) { - m_script = script - m_gitURL = '-' - m_revision = '-' - m_repoRoot = '' - m_cloneDepth = 1 - m_shallowClone = true - } - - void toolName(String name) { - m_toolName = name.trim() - } - - void toolRepo(String gitURL, String revision) { - m_gitURL = gitURL.trim() - m_revision = revision.trim() - } - - void repoRoot(String root) { - m_repoRoot = root.trim() - } - - void clonedepth(int depth) { - m_cloneDepth = depth - } - - void shallowclone(boolean shallow) { - m_shallowClone = shallow - } -} - -ToolOptions getNewToolOptions() { - return new ToolOptions(this) -} - -def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) -} - -def getTool(meta, ToolOptions options, String targetDir) { - gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) - - registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) -} - -return this diff --git a/Common_scripts/copy/all.groovy b/Common_scripts/copy/all.groovy deleted file mode 100644 index 24aacfb..0000000 --- a/Common_scripts/copy/all.groovy +++ /dev/null @@ -1,67 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - // Force workspace to be cleaned - CLEAN_WORKSPACE = "true" - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - } - } - } - stage('Import source artifacts') { - steps { - script { - def copyoptions = importer.getNewImportOptions() - copyoptions.source( - SOURCE_ARTIFACT_LOCATION, - SOURCE_ARTIFACT_BUILD_JOB, - SOURCE_ARTIFACT_BUILD_SELECTOR, - SOURCE_ARTIFACT_RELEASE_REPO, - SOURCE_ARTIFACT_RELEASE_REVISION) - echo "Nested: ${NESTED_COPY}" - copyoptions.setnested(NESTED_COPY == 'true') - importer.artifactimport(meta, copyoptions, "${WORKSPACE}/.") - } - } - } - stage('Archiving artifacts') { - steps { - script { - // Save the imported meta - while the unwanted meta is written to file then deleted - def stashname = 'Imported meta stash' - stash(allowEmpty: true, - includes: 'meta/**', - name: stashname, - useDefaultExcludes: false) - - def artifactlist = meta.generateartifacts(['**/*']) - cleaner.clean(['meta', 'autobuild']) - - unstash(name: stashname) - - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/copy/esw.groovy b/Common_scripts/copy/esw.groovy deleted file mode 100644 index b9eb737..0000000 --- a/Common_scripts/copy/esw.groovy +++ /dev/null @@ -1,67 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean exports') { - steps { - script { - env.ESW = 'esw' - env.ESWDEST = "${WORKSPACE}/${ESW}" - env.ESWEXPORT = 'embedded_software' - cleanfolders = [ - ESWDEST, - ESWEXPORT, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Copying embedded software artifacts') { - steps { - script { - def eswimportoptions = importer.getNewImportOptions() - eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) - eswimportoptions.setincrementversionnumber(true) - importer.artifactimport(meta, eswimportoptions, ESWDEST) - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) - sh """ - rm -v -rf diff - rm -v -rf meta - mv -v "${ESW}"/* . - """ - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} \ No newline at end of file diff --git a/Common_scripts/copy/filtered.groovy b/Common_scripts/copy/filtered.groovy deleted file mode 100644 index 3ad24cd..0000000 --- a/Common_scripts/copy/filtered.groovy +++ /dev/null @@ -1,73 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean') { - steps { - script { - env.IMPORT = "${WORKSPACE}/import" - env.FILTEREDDEST = "${WORKSPACE}/filtered" - cleanfolders = [ - FILTEREDDEST, - IMPORT, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Import artifacts') { - steps { - script { - def importoptions = importer.getNewImportOptions() - importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) - importer.artifactimport(meta, importoptions, IMPORT) - } - } - } - stage('Filtered copy') { - steps { - script { - dir(IMPORT) { - fileOperations([ - fileCopyOperation(excludes: EXCLUDES, - flattenFiles: FLATTENOUTPUT, - includes: INCLUDES, - targetLocation: FILTEREDDEST)]) - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts(["filtered/**"]) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/copy/logic.groovy b/Common_scripts/copy/logic.groovy deleted file mode 100644 index db9dbc3..0000000 --- a/Common_scripts/copy/logic.groovy +++ /dev/null @@ -1,79 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean exports') { - steps { - script { - LOGICDEST = 'logic' - cleanfolders = [ - LOGICDEST, - "${WORKSPACE}/embedded_logic", - "${WORKSPACE}/el", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Import source artifacts') { - steps { - script { - def logicoptions = importer.getNewImportOptions() - logicoptions.source( - LOGIC_ARTIFACT_LOCATION, - LOGIC_ARTIFACT_PROJECT, - LOGIC_ARTIFACT_BUILD_SELECTOR, - LOGIC_ARTIFACT_RELEASE_REPO, - LOGIC_ARTIFACT_RELEASE_REVISION) - importer.artifactimport(meta, logicoptions, LOGICDEST) - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts(['**/*']) - def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - - cleaner.clean(['meta', 'autobuild']) - - dir(LOGICDEST) { - fileOperations([ - fileCopyOperation(excludes: '', - flattenFiles: false, - includes: '**/*', - targetLocation: WORKSPACE) - ]) - } - - cleaner.clean([LOGICDEST]) - - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/differences/.project b/Common_scripts/differences/.project deleted file mode 100644 index 75af0fc..0000000 --- a/Common_scripts/differences/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - Release_difference - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/Common_scripts/differences/.pydevproject b/Common_scripts/differences/.pydevproject deleted file mode 100644 index c3434da..0000000 --- a/Common_scripts/differences/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - - -/${PROJECT_DIR_NAME} - -python 3.0 -Default - diff --git a/Common_scripts/differences/dometahierarchy.groovy b/Common_scripts/differences/dometahierarchy.groovy deleted file mode 100644 index f9a2404..0000000 --- a/Common_scripts/differences/dometahierarchy.groovy +++ /dev/null @@ -1,836 +0,0 @@ -import groovy.json.JsonSlurper - -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" - -sourcerepofolder = "${WORKSPACE}/sourcerepos" -tempfolder = "${WORKSPACE}/temp" - -reportfolder = "${WORKSPACE}/reports" - -class JiraTicketInfo { - String jiranumber - String uri - - JiraTicketInfo() { - jiranumber = '' - uri = '' - } - - def String dump() { - return "jiranumber: ${jiranumber} uri: ${uri}" - } -} - -class DifferenceRecord { - String productvariant - String subcomponentname - - String rawdifferencetext - String sha - String type - String mergeinfo - String squashinfo - List jiraticketinfo - String comment - - DifferenceRecord() { - productvariant = "" - subcomponentname = "" - - rawdifferencetext = "" - sha = "" - type = "" - mergeinfo = "" - squashinfo = "" - jiraticketinfo = [] - comment = "" - } - - def String dump() { - String jiraticketstring = '' - for (j in jiraticketinfo) { - jiraticketstring += j.dump() + " " - } - return "Difference Record:\nproductvariant: ${productvariant}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" - } -} - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } - dir(reportfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Release summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason += reason + "\n" - echo "Marked build as unstable: Reason: ${reason}" -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def showmetaitems(metaitems) { - String info = '' - for (key in metaitems.keySet()) { - def relatedmetainfo = metaitems[key] - def accumulatedpath = relatedmetainfo.accumulatedpath - info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" - } - echo info -} - -def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { - boolean checkedout = false - def String sourcerepo = buildmeta.gitinfo.remoteurl - if (sourcerepo?.trim()) { - echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" - if (sourcerepo) { - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } - } - checkedout = true - } - return checkedout -} - -def String getcomponent(meta) { - def String uniquefolder = "${meta.productvariant()}_${meta.productinfo.subcomponentname}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - - boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) - - return [checkedout, uniquefolderforsourcerepo] -} - -def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { - def String releaseexpression - if (tripledot) { - releaseexpression = earlyrevision + "..." + laterrevision - } else { - releaseexpression = earlyrevision + ".." + laterrevision - } - def String differencetext - dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() - } - return differencetext -} - -def extractsha(String line) { - def pattern = ~"^(\\p{XDigit}*) (.*)" - def match = (line =~ pattern) - match.find() - - def sha = match.group(1) - def restofline = match.group(2) - - //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" - - return [sha.trim(), restofline.trim()] -} - -def extractsquashes(String line) { - def pattern1 = "^(.*)(Squashed .*)" - def match1 = (line =~ pattern1) - if (match1.find()) { - return [true, match1.group(2)] - } - - return [false, line] -} - -def extractmerges(String line) { - def pattern1 = "^(.*)(Merge branch .*)" - def match1 = (line =~ pattern1) - if (match1.find()) { - return [true, match1.group(2)] - } - - def pattern2 = "^(.*)(Merge remote-tracking branch .*)" - def match2 = (line =~ pattern2) - if (match2.find()) { - return [true, match2.group(2)] - } - - def pattern3 = "^(.*)(Merge commit .*)" - def match3 = (line =~ pattern3) - if (match3.find()) { - return [true, match3.group(2)] - } - - def pattern4 = "^(.*)(Add .* from commit .*)" - def match4 = (line =~ pattern4) - if (match4.find()) { - return [true, match4.group(2)] - } - - return [false, line] -} - -def jiraticketexists(String jiraticket) { - def String URI = "${JIRA_URI}/browse/${jiraticket}" - def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" - def get = new URL(RestfulURI).openConnection(); - get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) - get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) - get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) - get.setRequestProperty( 'Accept', 'application/json' ) - def getRC = get.getResponseCode(); - - def String redirectedjiraticket = jiraticket - if(!getRC.equals(200)) { - return [false, '', jiraticket] - } else { - def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) - redirectedjiraticket = jsonresponse['key'] - } - return [true, URI, redirectedjiraticket] -} - -def extractjiraticketinfo(String line) { - def List jiratickets = [] - boolean exists = false - String URI = '' - - while (true) { - line = line.trim() - def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line - def commamatch = (line =~ commapattern) - if (commamatch.find()) { - line = commamatch.group(1).trim() - } - def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 - def match1 = (line =~ pattern1) - if (match1.find()) { - jiraticket = match1.group(1) - (exists, URI, jiraticket) = jiraticketexists(jiraticket) - if (exists) { - def JiraTicketInfo j = new JiraTicketInfo() - j.jiranumber = jiraticket - j.uri = URI - jiratickets.add(j) - line = match1.group(2) - } else { - break - } - } else { - def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 - def match2 = (line =~ pattern2) - if (match2.find()) { - jiraticket = match2.group(1) + "-" + match2.group(2) - (exists, URI, jiraticket) = jiraticketexists(jiraticket) - if (exists) { - def JiraTicketInfo j = new JiraTicketInfo() - j.jiranumber = jiraticket - j.uri = URI - jiratickets.add(j) - line = match2.group(3) - } else { - break - } - } else { - def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 - def match3 = (line =~ pattern3) - if (match3.find()) { - jiraticket = match3.group(1) + "-" + match3.group(2) - (exists, URI, jiraticket) = jiraticketexists(jiraticket) - if (exists) { - def JiraTicketInfo j = new JiraTicketInfo() - j.jiranumber = jiraticket - j.uri = URI - jiratickets.add(j) - line = match3.group(3) - } else { - break - } - } else { - break - } - } - } - } - boolean result = jiratickets.size() > 0 - return [result, jiratickets, line] -} - -def extractreverts(String line) { - boolean result = true - - def pattern1 = '^(.*)Revert \"(.*)\"' - def match1 = (line =~ pattern1) - if (match1.find()) { - def line2 = match1.group(2) - (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) - return [true, originaljiraticketinfo, originalcomment] - } - - def pattern2 = '^(.*)Partial Revert \"(.*)\"' - def match2 = (line =~ pattern2) - if (match2.find()) { - def line2 = match2.group(2) - (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) - return [true, originaljiraticketinfo, originalcomment] - } - - return [false, [], line] -} - -def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { - def String differencetext - dir(gitrootfolder) { - differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() - } - return differencetext -} - -def List fillsquashrecords(String sourcepath, String sha) { - def List differencerecords = [] - - String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) - int count = 0 - squashloglines.split('\n').each { - if (count > 0) { // Ignore first line - def line = it - if (line?.trim()) { - def List drs = fillrecord(meta, line.trim(), sourcepath) - differencerecords.addAll(drs) - //echo "${dr.dump()}" - } - } - count += 1 - } - return differencerecords -} - -def List fillrecord(meta, line, String sourcepath) { - def List differencerecords = [] - def DifferenceRecord dr = new DifferenceRecord() - - dr.productvariant = meta.productvariant() - dr.subcomponentname = meta.productinfo.subcomponentname - - dr.rawdifferencetext = line - - // Example of raw text to process: - // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py - - (sha, line2) = extractsha(line) - dr.sha = sha - - (result, squashinfo) = extractsquashes(line2) - if (result) { - dr.type = "Squash" - dr.squashinfo = squashinfo - squashrecords = fillsquashrecords(sourcepath, sha) - differencerecords.addAll(squashrecords) - } else { - (result, mergeinfo) = extractmerges(line2) - if (result) { - dr.type = "Merge" - dr.mergeinfo = mergeinfo - } else { - (result, jiraticketinfo, comment) = extractreverts(line2) - if (result) { - dr.type = "Revert" - dr.jiraticketinfo = jiraticketinfo - dr.comment = comment - } else { - (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) - if (result) { - dr.type = "Ticketed" - dr.jiraticketinfo = jiraticketinfo - dr.comment = comment - } else { - dr.type = "Unticketed" - dr.comment = line2 - } - } - } - } - differencerecords.add(dr) - return differencerecords -} - -def List processtext(meta, String text, String sourcepath) { - def List differencerecords = [] - - text.split('\n').each { - def line = it - if (line?.trim()) { - def List drs = fillrecord(meta, line.trim(), sourcepath) - differencerecords.addAll(drs) - //echo "${dr.dump()}" - } - } - return differencerecords -} - -def getdifferencerecords(earlymeta, latermeta, String sourcepath) { - boolean result = true; - def List differencerecords = [] - - def earlygitinfo = earlymeta.gitinfo - def latergitinfo = latermeta.gitinfo - - if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { - def earlyrevision = earlygitinfo.gitrevision - def laterrevision = latergitinfo.gitrevision - echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" - summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") - def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) - differencerecords = processtext(earlymeta, differencetext, sourcepath) - if (earlyrevision == laterrevision) { - summaryappend(" No change") - } - } else { - summaryappend("The early and later git urls don't match") - echo "The early and later git urls don't match" - result = false; - setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") - } - - return [result, differencerecords] -} - -def deletedcomponent(earlymetaitems, key) { - echo "Component ${key} has been deleted in later version" - summaryappend("Deleted ") -} - -def addedcomponent(latermetaitems, key) { - echo "Component ${key} has been added in later version" - summaryappend("Added ") -} - -class SquashReportRecord { - String comment - int duplicates - - SquashReportRecord() { - comment = '' - duplicates = 0 - } -} - -def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { - dir(reportsubfolder) { - def String squashlist = "${title}\nSquash list:\n" - def squashessorted = squashes.sort() - for (asquash in squashessorted) { - def SquashReportRecord s = asquash.value - squashlist += "${s.comment} (${s.duplicates})\n" - } - writeFile(file: "squashlist.txt", text: squashlist) - } -} - -class MergeReportRecord { - String comment - int duplicates - - MergeReportRecord() { - comment = '' - duplicates = 0 - } -} - -def mergereport(String reportsubfolder, String title, LinkedHashMap merges, boolean iscomponent = true) { - dir(reportsubfolder) { - def String mergelist = "${title}\nMerge list:\n" - def mergessorted = merges.sort() - for (amerge in mergessorted) { - def MergeReportRecord m = amerge.value - mergelist += "${m.comment} (${m.duplicates})\n" - } - writeFile(file: "mergelist.txt", text: mergelist) - } -} - -class CommentsRecord { - String comment - int duplicates - - CommentsRecord() { - comment = '' - duplicates = 0 - } -} - -class RevertsReportRecord { - String jiraticket - String ticketURI - LinkedHashMap commentscollection - - RevertsReportRecord() { - jiraticket = '' - ticketURI = '' - commentscollection = new LinkedHashMap() - } -} - -def revertsreport(String reportsubfolder, String title, LinkedHashMap reverts, boolean iscomponent = true) { - dir(reportsubfolder) { - def String revertlist = "${title}\nRevert list:\n" - def revertssorted = reverts.sort() - for (arevert in revertssorted) { - def RevertsReportRecord r = arevert.value - revertlist += "${r.jiraticket} (${r.ticketURI}):\n" - def commentscollection = r.commentscollection - def sortedcommentscollection = commentscollection.sort() - for (commentrecord in sortedcommentscollection) { - def CommentsRecord c = commentrecord.value - revertlist += "${c.comment} (${c.duplicates})\n" - } - revertlist += "\n" - } - writeFile(file: "revertlist.txt", text: revertlist) - } -} - -class TicketedReportRecord { - String jiraticket - String ticketURI - LinkedHashMap commentscollection - - TicketedReportRecord() { - jiraticket = '' - ticketURI = '' - commentscollection = new LinkedHashMap() - } -} - -def ticketedreport(String reportsubfolder, String title, LinkedHashMap ticketed, boolean iscomponent = true) { - dir(reportsubfolder) { - def String ticketedlist = "${title}\nTicketed list:\n" - def ticketedsorted = ticketed.sort() - for (aticketed in ticketedsorted) { - def TicketedReportRecord r = aticketed.value - ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" - def commentscollection = r.commentscollection - def sortedcommentscollection = commentscollection.sort() - for (commentrecord in sortedcommentscollection) { - def CommentsRecord c = commentrecord.value - ticketedlist += "${c.comment} (${c.duplicates})\n" - } - ticketedlist += "\n" - } - writeFile(file: "ticketedlist.txt", text: ticketedlist) - } -} - -class UnticketedReportRecord { - String comment - int duplicates - - UnticketedReportRecord() { - comment = '' - duplicates = 0 - } -} - -def unticketedreport(String reportsubfolder, String title, LinkedHashMap unticketed, boolean iscomponent = true) { - dir(reportsubfolder) { - def String unticketedlist = "${title}\nUnticketed list:\n" - def unticketedsorted = unticketed.sort() - for (anunticketed in unticketedsorted) { - UnticketedReportRecord u = anunticketed.value - unticketedlist += "${u.comment} (${u.duplicates})\n" - } - writeFile(file: "unticketedlist.txt", text: unticketedlist) - } -} - -class TicketsReportRecord { - String jiraticket - String ticketURI - int duplicates - - TicketsReportRecord() { - jiraticket = '' - ticketURI = '' - duplicates = 0 - } -} - -def ticketsreport(String reportsubfolder, String title, tickets, boolean iscomponent = true) { - dir(reportsubfolder) { - String ticketlist = "${title}\nTicket list:\n" - ticketssorted = tickets.sort() - for (ticketnumber in ticketssorted) { - TicketsReportRecord t = ticketnumber.value - ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" - } - writeFile(file: "ticketlist.txt", text: ticketlist) - } -} - -def differencerecordsreport(String reportsubfolder, String title, differencerecords, boolean iscomponent = true) { - dir(reportsubfolder) { - String recordlist = "${title}\n" - for (dr in differencerecords) { - recordlist += dr.dump() + "\n\n" - } - writeFile(file: "recordlist.txt", text: recordlist) - } -} - -def boolean report(meta, title, differencerecords, boolean iscomponent = true) { - def Map ticketed = [:] - def Map squashes = [:] - def Map merges = [:] - def Map reverts = [:] - def Map unticketed = [:] - def Map tickets = [:] - - for (dr in differencerecords) { - if (dr.type == "Squash") { - def SquashReportRecord s - if (!squashes.containsKey(dr.mergeinfo)) { - s = new SquashReportRecord() - s.comment = dr.squashinfo - s.duplicates = 1 - } else { - s = squashes[dr.squashinfo] - s.duplicates += 1 - } - squashes[dr.squashinfo] = s - } - if (dr.type == "Merge") { - def MergeReportRecord m - if (!merges.containsKey(dr.mergeinfo)) { - m = new MergeReportRecord() - m.comment = dr.mergeinfo - m.duplicates = 1 - } else { - m = merges[dr.mergeinfo] - m.duplicates += 1 - } - merges[dr.mergeinfo] = m - } - if (dr.type == "Revert") { - def RevertsReportRecord r - for (ticket in dr.jiraticketinfo) { - JiraTicketInfo j = ticket - if (!reverts.containsKey(j.jiranumber)) { - r = new RevertsReportRecord() - r.jiraticket = j.jiranumber - r.ticketURI = j.uri - CommentsRecord c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - r.commentscollection[dr.comment] = c - } else { - r = reverts[j.jiranumber] - def CommentsRecord c - if (!r.commentscollection.containsKey(dr.comment)) { - c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - r.commentscollection[dr.comment] = c - } else { - c = r.commentscollection[dr.comment] - c.duplicates += 1 - } - r.commentscollection[dr.comment] = c - } - reverts[j.jiranumber] = r - def TicketsReportRecord t - if (!tickets.containsKey(j.jiranumber)) { - t = new TicketsReportRecord() - t.jiraticket = j.jiranumber - t.ticketURI = j.uri - t.duplicates = 1 - } else { - t = tickets[j.jiranumber] - t.duplicates += 1 - } - tickets[j.jiranumber] = t - } - } - if (dr.type == "Ticketed") { - def TicketedReportRecord t - for (ticket in dr.jiraticketinfo) { - JiraTicketInfo j = ticket - if (!ticketed.containsKey(j.jiranumber)) { - t = new TicketedReportRecord() - t.jiraticket = j.jiranumber - t.ticketURI = j.uri - CommentsRecord c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - t.commentscollection[dr.comment] = c - } else { - t = ticketed[j.jiranumber] - def CommentsRecord c - if (!t.commentscollection.containsKey(dr.comment)) { - c = new CommentsRecord() - c.comment = dr.comment - c.duplicates = 1 - t.commentscollection[dr.comment] = c - } else { - c = t.commentscollection[dr.comment] - c.duplicates += 1 - } - t.commentscollection[dr.comment] = c - } - ticketed[j.jiranumber] = t - def TicketsReportRecord ti - if (!tickets.containsKey(j.jiranumber)) { - ti = new TicketsReportRecord() - ti.jiraticket = j.jiranumber - ti.ticketURI = j.uri - ti.duplicates = 1 - } else { - ti = tickets[j.jiranumber] - ti.duplicates += 1 - } - tickets[j.jiranumber] = ti - } - } - if (dr.type == "Unticketed") { - UnticketedReportRecord u - if (!unticketed.containsKey(dr.comment)) { - u = new UnticketedReportRecord() - u.comment = dr.comment - u.duplicates = 1 - } else { - u = unticketed[dr.comment] - u.duplicates += 1 - } - unticketed[dr.comment] = u - } - } - String reportsubfolder = "" - if (iscomponent) { - reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" - } else { - reportsubfolder = "${reportfolder}/Combined" - } - mergereport(reportsubfolder, title, merges, iscomponent) - squashreport(reportsubfolder, title, squashes, iscomponent) - revertsreport(reportsubfolder, title, reverts, iscomponent) - ticketedreport(reportsubfolder, title, ticketed, iscomponent) - unticketedreport(reportsubfolder, title, unticketed, iscomponent) - ticketsreport(reportsubfolder, title, tickets, iscomponent) - differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) -} - -def title(earlymeta, latermeta) { - return """ - Differences in: - Product: ${earlymeta.productvariant()} Subcomponent: ${earlymeta.productinfo.subcomponentname} - between: - then: - Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} - now: - Build date: ${latermeta.jenkinsbuildinfo.builddate} Build stamp: ${latermeta.buildstampinfo.buildstamp} Git revision: ${latermeta.gitinfo.gitrevision} - """ -} - -def compare(earlymetaitems, latermetaitems, key) { - boolean result = true; - echo "Collecting changes between early and later versions of component: ${key}" - - def earlymeta = earlymetaitems[key].metainfo - def latermeta = latermetaitems[key].metainfo - - summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: Changed: ") - - def boolean repoavailable - def String uniquefolderforsourcerepo - (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) - - List differencerecords = [] - if (repoavailable) { - (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) - - summaryappend(" Items: ${differencerecords.size()} ") - report(earlymeta, title(earlymeta, latermeta), differencerecords) - } else { - summaryappend(" No source repo ") - } - summaryappend("\n") - - return [result, differencerecords]; -} - -def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { - def List differencerecords = [] - - def int md = maxdepth.toInteger() - resetfolders() - preconditions() - - echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - - def earlymeta = meta.importrelated(true, earlymetafolder) - def latermeta = meta.importrelated(true, latermetafolder) - - def earlymetaitems = earlymeta.releasewalkerunique(md, '', earlymetafolder) - def latermetaitems = latermeta.releasewalkerunique(md, '', latermetafolder) - - echo "Early items:" - showmetaitems(earlymetaitems) - - echo "Later items:" - showmetaitems(latermetaitems) - - for (key in earlymetaitems.keySet()) { - if (latermetaitems.containsKey(key)) { - (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) - differencerecords = differencerecords + componentdifferencerecords - } else { - summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: ") - deletedcomponent(earlymetaitems, key) - summaryappend("\n") - } - } - for (key in latermetaitems.keySet()) { - if (!earlymetaitems.containsKey(key)) { - summaryappend("${latermeta.productvariant()} ${latermeta.productinfo.subcomponentname}: ") - addedcomponent(latermetaitems, key) - summaryappend("\n") - } - } - report(meta, title(earlymeta, latermeta), differencerecords, false) - return [summarystring, markunstable, unstablereason] -} - -return this; diff --git a/Common_scripts/differences/file.groovy b/Common_scripts/differences/file.groovy deleted file mode 100644 index ee1b8b3..0000000 --- a/Common_scripts/differences/file.groovy +++ /dev/null @@ -1,24 +0,0 @@ -def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { - echo "Comparing file: ${importedfile} with: ${currentfile} option: ${ignorewhitespace}" - def wsparameter = "-w" - if (!ignorewhitespace) { - wsparameter = "" - } - if (isUnix()) { - sh """ - diff -c ${wsparameter} "${importedfile}" "${currentfile}" - """ - } - else { - // diffutils can be installed on windows with chocolatey - powershell """ - # echo Checking utilities exist - # where diff - # if errorlevel 1 EXIT /b 1 - - diff.exe -c ${wsparameter} "${importedfile}" "${currentfile}" - """ - } -} - -return this; diff --git a/Common_scripts/differences/json.groovy b/Common_scripts/differences/json.groovy deleted file mode 100644 index dbf47b8..0000000 --- a/Common_scripts/differences/json.groovy +++ /dev/null @@ -1,51 +0,0 @@ -def comparefiles(importedjsonfile, currentjsonfile) { - echo "Comparing json file: ${importedjsonfile} with: ${currentjsonfile}" - if (isUnix()) { - sh """ - which jq - if [ \$? -ne 0 ]; then exit 1; fi - # jq -S sorts json output - diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") - """ - } - else { - // Jq and diffutils can be installed on windows with chocolatey - powershell """ - # echo Checking utilities exist - # set - # echo %PATH% - try { - if (Get-Command jq) {} - } - catch { - echo "jq is unavailable" - exit \$false - } - try { - if (Get-Command diff) {} - } - catch { - echo "diff is unavailable" - exit \$false - } - - # echo Sorting json in files to compare - # jq -S sorts json output - jq -S . "${importedjsonfile}" > temp1.txt - if (\$lastexitcode -ge 1) { exit 1 } - jq -S . "${currentjsonfile}" > temp2.txt - if (\$lastexitcode -ge 1) { exit 1 } - - # echo Diffing sorted json files - diff.exe -c temp1.txt temp2.txt - \$lec = \$lastexitcode - - Remove-Item temp1.txt - Remove-Item temp2.txt - - if (\$lec -ge 1) { exit 1 } - """ - } -} - -return this; diff --git a/Common_scripts/differences/metahierarchy.groovy b/Common_scripts/differences/metahierarchy.groovy deleted file mode 100644 index 01c1890..0000000 --- a/Common_scripts/differences/metahierarchy.groovy +++ /dev/null @@ -1,102 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - SUMMARYDIR = "${WORKSPACE}/summary" - REPORTDIR = "${WORKSPACE}/reports" - EARLYROOTDIR = 'early' - EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" - LATERROOTDIR = 'later' - LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" - - cleanfolders = [ - "${WORKSPACE}/build_artifacts", - "${EARLYDIR}", - "${LATERDIR}", - "${SUMMARYDIR}", - "${REPORTDIR}", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Obtain top level meta') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - - def earlyimportoptions = importer.getNewImportOptions() - earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) - earlyimportoptions.setincrementversionnumber(false) - earlyimportoptions.setfilter('meta/**') - importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) - - def laterimportoptions = importer.getNewImportOptions() - laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) - laterimportoptions.setincrementversionnumber(false) - laterimportoptions.setfilter('meta/**') - importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) - } - } - } - stage('Diff components') { - steps { - script { - def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") - (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,reports/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/documentation/doxygen_deploy.groovy b/Common_scripts/documentation/doxygen_deploy.groovy deleted file mode 100644 index e2f74a7..0000000 --- a/Common_scripts/documentation/doxygen_deploy.groovy +++ /dev/null @@ -1,80 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Clean imports') { - steps { - script { - env.DOXYGEN = "doxygen" - cleanfolders = [ - DOXYGEN, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Import doxygen artifacts') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def importoptions = importer.getNewImportOptions() - - importoptions.jenkinsjob(BUILD_JOB_WITH_DOXYGEN_ARTIFACTS, BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR) - importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') - doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) - - productvariant = doxygenmeta.productvariant() - buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix - - echo "Doyxgen imported for product: ${productvariant} and build stamp prefix: ${buildstampprefix}" - } - } - } - stage('Copy doxygen to server') { - steps { - script { - doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" - pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productvariant}/${buildstampprefix}" - echo "Copying from ${doxygensource} to ${pathonserver}" - if (isUnix()) { - sh """#!/bin/bash - rm -rfv "${pathonserver}" - if [ \$? -ne 0 ]; then exit 1; fi - - mkdir -pv "${pathonserver}" - if [ \$? -ne 0 ]; then exit 1; fi - - echo mv -v "${doxygensource}"/ "${pathonserver}" - mv -v "${doxygensource}"/ "${pathonserver}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } else { - error("This job does not run on Windows at the moment") - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/hardware/sominfo.groovy b/Common_scripts/hardware/sominfo.groovy deleted file mode 100644 index 72a3ca9..0000000 --- a/Common_scripts/hardware/sominfo.groovy +++ /dev/null @@ -1,80 +0,0 @@ -def sominfostore = null - -/* Example expected JSON -{ - "SomType": { - "Som_BA4": { - "Artifact info": { - "Location": "Jenkins job artifact", - "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", - "Job selector": "true", - "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", - "Release repo revision": "" - } - }, - "Som_FC5": { - "Artifact info": { - "Location": "Jenkins job artifact", - "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", - "Job selector": "true", - "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", - "Release repo revision": "" - } - } - } -} -*/ - -def String asJsonText() { - return writeJSON(returnText: true, json: sominfostore, pretty: 1) -} - -def boolean exists() { - return (sominfostore != null) -} - -def set(String jsontext) { - sominfostore = readJSON(text: jsontext) - echo "Som support info:\n${asJsonText()}" -} - -def get() { - return sominfostore -} - -def somtypes() { - return sominfostore['SomType'] -} - -def artifactinfo(somtype) { - return somtype['Artifact info'] -} - -def artifactlocation(artifactinfo) { - return artifactinfo['Location'] -} - -def boolean isjoblocation(artifactinfo) { - return (artifactinfo['Location'] == 'Jenkins job artifact' || - artifactinfo['Location'] == 'Jenkins job artefact') -} - -def boolean isreleaserepolocation(artifactinfo) { - return artifactinfo['Location'] == 'Release repository' -} - -def artifactjob(artifactinfo) { - return [ - "name": artifactinfo['Job'], - "selector": "${artifactinfo['Job selector']}" - ] -} - -def releaserepo(artifactinfo) { - return [ - "repo": artifactinfo['Release repo'], - "revision": artifactinfo['Release repo revision'] - ] -} - -return this \ No newline at end of file diff --git a/Common_scripts/import/importer.groovy b/Common_scripts/import/importer.groovy deleted file mode 100644 index fa32ebf..0000000 --- a/Common_scripts/import/importer.groovy +++ /dev/null @@ -1,408 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - -class ImportOptions { - def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) - - boolean m_incrementversionnumber // If true, increment the parent's (4th) version number with the (4th) version number of this object being imported - String m_filter // Only import files/paths/folders that match the filter - String m_excludes // Exclude files/paths/folders that match the exclude filter - boolean m_fingerprint // Do Jenkins fingerprinting on imported artifacts - boolean m_nested // Import all child artifacts too (identified in the related meta info - boolean m_flatten // Flatten files and folders imported - boolean m_includemetainfilter // Make sure meta is also included in filter - boolean m_clean // Clean import folder before obtaining artifacts - boolean m_appendtoparentmeta // Append to parent meta - String m_sourceselect // Where are the artifacts coming from - - String m_jobname - def m_buildnumberorselector - - String m_repo - String m_revision - - boolean m_showdetails - - ImportOptions(script, - String sourceselect = 'Jenkins job artifact', - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false, - boolean nested = false, - boolean flatten = false, - boolean includemetainfilter = true, - boolean clean = true, - appendtoparentmeta = true, - showdetails = false) { - m_script = script - m_sourceselect = sourceselect.trim() - m_incrementversionnumber = incrementversionnumber - m_filter = filter.trim() - m_excludes = excludes.trim() - m_fingerprint = fingerprint - m_nested = nested - m_flatten = flatten - m_includemetainfilter = includemetainfilter - m_clean = clean - m_appendtoparentmeta = appendtoparentmeta - m_showdetails = showdetails - } - - String show() { - return """m_incrementversionnumber: ${m_incrementversionnumber} - m_filter: ${m_filter} - m_excludes: ${m_excludes} - m_fingerprint: ${m_fingerprint} - m_nested: ${m_nested} - m_flatten: ${m_flatten} - m_includemetainfilter: ${m_includemetainfilter} - m_clean: ${m_clean} - m_appendtoparentmeta: ${m_appendtoparentmeta} - m_sourceselect: ${m_sourceselect} - m_jobname: ${m_jobname} - m_buildnumberorselector: ${m_buildnumberorselector} - m_repo: ${m_repo} - m_revision: ${m_revision} - """ - } - - void setsourceselect(String sourceselect) { - sourceselect = sourceselect.trim() - switch (sourceselect) { - case 'Jenkins job artefact': - m_sourceselect = 'Jenkins job artifact' - break - case 'Jenkins job artifact': - case 'Release repository': - m_sourceselect = sourceselect - break - default: - m_script.error("Unknown type of artifact source select: ${sourceselect}") - } - } - - void setincrementversionnumber(boolean incrementversionnumber = true) { - m_incrementversionnumber = incrementversionnumber - } - - void setfilter(String filter) { - m_filter = filter.trim() - } - - void setexcludes(String excludes) { - m_excludes = excludes.trim() - } - - void setfingerprint(boolean fingerprint = true) { - m_fingerprint = fingerprint - } - - void setnested(boolean nested = true) { - m_nested = nested - } - - void flattenimports(boolean flatten = true) { - m_flatten = flatten - } - - void includemetainfilter(boolean includemetainfilter = true) { - m_includemetainfilter = includemetainfilter - } - - void clean(boolean clean = true) { - m_clean = clean - } - - void setappendtoparentmeta(boolean appendtoparentmeta) { - m_appendtoparentmeta = appendtoparentmeta - } - - void jenkinsjob(String jobname, String buildnumberorselector) { - m_jobname = jobname.trim() - m_buildnumberorselector = [ - $class: 'ParameterizedBuildSelector', - parameterName: buildnumberorselector - ] - setsourceselect('Jenkins job artifact') - } - - void jenkinsjob(String jobname, int buildnumberorselector) { - m_jobname = jobname.trim() - m_buildnumberorselector = [ - $class: 'ParameterizedBuildSelector', - parameterName: "${buildnumberorselector}" - ] - setsourceselect('Jenkins job artifact') - } - - void releaserepo(String repo, String revision) { - m_repo = repo.trim() - m_revision = revision.trim() - setsourceselect('Release repository') - } - - void source(String sourceselect, String jobname, String buildnumberorselector, String repo, String revision) { - jenkinsjob(jobname, buildnumberorselector) - releaserepo(repo, revision) - setsourceselect(sourceselect) - } - - void showdetails(boolean showdetails) { - m_showdetails = showdetails - } -} - -ImportOptions getNewImportOptions() { - return new ImportOptions(this) -} - -def formFilter(ImportOptions importoptions) { - def String filter = importoptions.m_filter - if (filter != '') { - if (importoptions.m_includemetainfilter) { - filter = 'meta/**,' + filter - } - } - echo "Import options filter is: ${filter}" - return filter -} - -def conditionalClean(ImportOptions importoptions, String destpath) { - if (importoptions.m_clean) { - echo "Cleaning ${destpath}" - dir(destpath) { - deleteDir() - } - } -} - -def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_sourceselect != 'Jenkins job artifact') { - error("Import options settings have not been set for: Jenkins job artifact") - return - } - - conditionalClean(importoptions, destpath) - - echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" - echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" - copyArtifacts( - filter: formFilter(importoptions), - excludes: importoptions.m_excludes, - fingerprintArtifacts: importoptions.m_fingerprint, - projectName: importoptions.m_jobname, - selector: importoptions.m_buildnumberorselector, - flatten: importoptions.m_flatten, - target: destpath - ) - def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath) - if (importoptions.m_incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) - } - return importedmeta -} - -def rewritemetafileasreleased(meta, String filepath) { - // Modify the meta info to say that is has already been released - meta.markreleased() - meta.saveJsonSerialised(filepath) -} - -def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_sourceselect != 'Release repository') { - error("Import options settings have not been set for: Release repository") - return - } - - def rooturl = importoptions.m_repo - def rootrevision = importoptions.m_revision - echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - - conditionalClean(importoptions, destpath) - - def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) - gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - - def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) - - echo "Getting associated metadata from path: ${destpath}" - def collectedmetalist = importedmeta.getassociated(destpath) - for(def collectedmeta in collectedmetalist) { - echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}" - } - - echo "Cloning associated repos inside path: ${destpath}" - for(def collectedmeta in collectedmetalist) { - def relatedmeta = collectedmeta.metainfo - def relatedoffsetpath = collectedmeta.accumulatedpath - def cloneurl = relatedmeta.releaseinfo.releaserepoURL - def clonerevision = "refs/tags/${relatedmeta.releasetag()}" - def clonepath = "${destpath}/${relatedoffsetpath}" - if (relatedmeta.componentidentifier() != '_' && - cloneurl != '') { - echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) - gh.stripgithistory(clonepath) - def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" - echo "Marking related meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(relatedmeta, metafilepath) - } else { - echo "Not importing component that would be placed at path: ${clonepath}" - } - } - gh.stripgithistory(destpath) - def metafilepath = "${destpath}/${meta.metafilepath()}" - echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(importedmeta, metafilepath) - - if (importoptions.m_incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) - } - return importedmeta -} - -def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_sourceselect != 'Release repository') { - error("Import options settings have not been set for: Release repository") - return - } - - conditionalClean(importoptions, destpath) - - def rooturl = importoptions.m_repo - def rootrevision = importoptions.m_revision - echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - - def ArrayList sparseitemlist = gh.convertSparseItemsString(formFilter(importoptions)) - gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - - def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) - - gh.stripgithistory(destpath) - def metafilepath = "${destpath}/${meta.metafilepath()}" - echo "Marking meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(importedmeta, metafilepath) - - if (importoptions.m_incrementversionnumber) { - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) - } - return importedmeta -} - -def artifactimport(meta, ImportOptions importoptions, String destpath) { - if (importoptions.m_showdetails) { - echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" - } - switch (importoptions.m_sourceselect) { - case 'Jenkins job artifact': - case 'Jenkins job artefact': - return artifactimportfrombuildjob(meta, importoptions, destpath) - case 'Release repository': - if (importoptions.m_nested) { - return hierarchyartifactimportfromrepo(meta, importoptions, destpath) - } - return artifactimportfromrepo(meta, importoptions, destpath) - default: - error("Unknown type of artifact source: ${source}") - } - return meta -} - -// Deprecated (use above) -def artifactimport(meta, - String sourceselect, - String jenkinsjob, - buildnumberorselector, - String repo, - String revision, - String destpath, - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false, - boolean nested = false) { - echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" - def importoptions = importer.getNewImportOptions() - - importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - importoptions.setnested(nested) - - return artifactimport(meta, importoptions, destpath) -} - -// Deprecated (use above) -def artifactimport(meta, - String buildjob, - int buildnumber, - String destpath, - boolean incrementversionnumber = true, - String filter = '**', - String excludes = '', - boolean fingerprint = false) { - echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" - def importoptions = importer.getNewImportOptions() - - importoptions.jenkinsjob(buildjob, " ${buildnumber}") - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - - return artifactimport(meta, importoptions, destpath) -} - -// Deprecated (use above) -def artifactimport(meta, - String buildjob, - buildnumberorselector, - String destpath, - boolean incrementversionnumber = true, - String filter = '', - String excludes = '', - boolean fingerprint = false) { - echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" - def importoptions = importer.getNewImportOptions() - - importoptions.jenkinsjob(buildjob, buildnumberorselector) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(filter) - importoptions.setexcludes(excludes) - importoptions.setfingerprint(fingerprint) - - return artifactimport(meta, importoptions, destpath) -} - -// Import the artifacts of a named child component found below parent meta's hierarchy -def artifactimport(meta, - parentmeta, - String parentmetapath, - String childproduct, - String childsubcomponent, - ImportOptions importoptions, - String destinationfolder) { - // Import from parent meta - def childmeta = meta.getNewMetaObject() - def childmeta2 - echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" - echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" - def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) - if (findresults.matchfound) { - childmeta = findresults.matchedmetaitem.metainfo - if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) { - importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) - } else { - importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) - } - childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) - } else { - error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") - } - return childmeta2 -} - -return this; diff --git a/Common_scripts/install/installhelper.groovy b/Common_scripts/install/installhelper.groovy deleted file mode 100644 index 8dbf23e..0000000 --- a/Common_scripts/install/installhelper.groovy +++ /dev/null @@ -1,136 +0,0 @@ -import hudson.FilePath - -// Transfer file from intermediate position to final position; -// creating necessary folders on the way; -// deleting intermediate file -def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { - def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() - - def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" - def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" - def export_filepath = new hudson.FilePath(channel, abs_export_out) - def export_out_folder = export_filepath.getParent() - def export_out_folder_string = export_out_folder.toString() - def export_out_file = export_filepath.getName() - - echo "Creating folder ${export_out_folder_string}" - export_out_folder.mkdirs() - - echo "Copying from ${pathToIntermediate} to ${abs_export_out}" - def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) - intermediate_filepath.copyTo(export_filepath) - intermediate_filepath.delete() -} - -// Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { - artifactFetched = [:] - - if (showdetails) { - echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") - } - def importoptions = importer.getNewImportOptions() - importoptions.showdetails(showdetails) - for (installer in toBuild) { - def installerinfo = installerDetails.get(installer) - if (showdetails) { - echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") - } - def artifactKeys = installerinfo[1] - if (artifactKeys instanceof java.lang.String) { - artifactKeys = [ artifactKeys ] - } - if (showdetails) { - echo ("artifactKeys:\n${artifactKeys}") - } - for (artifactKey in artifactKeys) { - artifactDetail = artifactDetails.get(artifactKey) - if (showdetails) { - echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") - } - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail - if (artifactFetched.get(artifactKey) == null) { - importoptions.jenkinsjob(sourceProject, sourceselector) - importoptions.setincrementversionnumber(incrementversionnumber) - importoptions.setfilter(toCopy) - importoptions.setexcludes(notToCopy) - - def importmeta = importer.artifactimport(meta, importoptions, destFolder) - artifactFetched.put(artifactKey, importmeta) - } - } - } - if (showdetails) { - echo ("artifactFetched:\n${artifactFetched}") - } - return artifactFetched -} - -@NonCPS -def checkISMfilebackend(String xml) { - def String message = "" - - def parsedxml = new XmlParser(true, true, true).parseText(xml) - def pathvariableinfo = [:] - def exitcode = 0 - // Load up a map of path variable and path pairs - // Ignore blank path definitions - // Ignore the workspace definition which should have the only absolute path - parsedxml.each{ table -> - if (table.@name == "ISPathVariable") { - table['row'].each{ row -> - def pathname = row['td'][0].value()[0] - def path = row['td'][1].value()[0] - if (path && pathname != 'WORKSPACE') { - pathvariableinfo[pathname] = path - } - } - } - } - // Check that each path variable is defined beginning with a path variable or the WORKSPACE one - pathvariableinfo.each{ pair -> - message += "pair key: ${pair.key} value: ${pair.value}\n" - def path = pair.value - def pathvarmatcher = path =~ /<(.*)>.*/ - if (pathvarmatcher.matches()) { - def pathvarfound = pathvarmatcher.group(1) - if (pathvarfound != 'WORKSPACE') { - if (!pathvariableinfo.containsKey(pathvarfound)) { - message += "Undefined path variable ${pathvarfound} found\n" - exitcode = 1 - } - } - } else { - message += "Path: ${path} does not begin with a path variable\n" - exitcode = 1 - } - } - return [exitcode, message] -} - -def checkISMfile(String ISMfile) { - echo "Checking ISM file: ${ISMfile}" - def xml = readFile(ISMfile) - //echo "ISM file xml: ${xml}" - (exitcode, message) = checkISMfilebackend(xml) - echo message - if (exitcode > 0) { - error "ISM file errors found" - } else { - echo "Checked ISM file - ok" - } -} - -certificatethumbprints = [:] -certificatethumbprints['floreana'] = '0FDEB0A227982C877146CA595091173AC9655479' // Authoring takes place on this machine - so this is what should be in the ISM file already -certificatethumbprints['hawaii'] = '49FD1B361CFF35014B81940C5B046543045DD34B' - -def selectSigningCertificate(String nodeName, String ISMfile) { - def masterpattern = '0FDEB0A227982C877146CA595091173AC9655479' - echo "Substituting occurrences of ${masterpattern} with ${certificatethumbprints[nodeName]} in ISM file: ${ISMfile}" - - textutils_subst = load("${autobuilder.commonscripts()}/textutils/subst.groovy") - textutils_subst.substitute(ISMfile, masterpattern, certificatethumbprints[nodeName]) -} - -return this; diff --git a/Common_scripts/meta/folder.groovy b/Common_scripts/meta/folder.groovy deleted file mode 100644 index 34448dd..0000000 --- a/Common_scripts/meta/folder.groovy +++ /dev/null @@ -1,1201 +0,0 @@ -import java.nio.file.Paths -import java.nio.file.Path -import groovy.json.JsonSlurperClassic - -abstract class GISJsonSerialised implements Serializable { - def script - - GISJsonSerialised(script) { - this.script = script - } - - abstract LinkedHashMap serialisationdata() - - abstract void deserialiseJson(rawmeta) - - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) - } -} - -class ProductInfo extends GISJsonSerialised { - private String format - private String productname - private String subcomponentname - private String variant - private String uniqueid - - ProductInfo(script, String productname = '', String subcomponentname = '', String variant = '') { - super(script) - this.format = '0.1' - this.productname = productname - this.subcomponentname = subcomponentname - this.variant = variant - this.uniqueid = UUID.randomUUID().toString() - } - - void set(String productname, String subcomponentname, String variant = '') { - // Make sure the fields can be used in git tags - def int pstatus = 0 - if (script.isUnix()) { - pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) - } - else { - pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) - } - if (pstatus > 0) { - script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") - } - def int sstatus = 0 - if (script.isUnix()) { - sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) - } - else { - sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) - } - if (sstatus > 0) { - script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") - } - if (variant) { - def int vstatus = 0 - if (script.isUnix()) { - vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) - } - else { - vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) - } - if (vstatus > 0) { - script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") - } - } - this.productname = productname - this.subcomponentname = subcomponentname - this.variant = variant - } - - void set(ProductInfo otherinfo) { - this.productname = otherinfo.productname - this.subcomponentname = otherinfo.subcomponentname - this.variant = otherinfo.variant - this.uniqueid = otherinfo.uniqueid - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - productname = rawmeta['productname'] - subcomponentname = rawmeta['subcomponentname'] - variant = rawmeta['variant'] ?: '' - uniqueid = rawmeta['uniqueid'] ?: uniqueid - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'productname' : productname, - 'subcomponentname' : subcomponentname, - 'variant' : variant, - 'uniqueid': uniqueid, - ] - return data - } -} - -class GitInfo extends GISJsonSerialised { - private String format - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo - - GitInfo(script, sourcepath = '', sourcerevision = '', gitrevision = '', remoteurl = '', remoteinfo = '') { - super(script) - this.format = '0.1' - this.sourcepath = sourcepath - this.sourcerevision = sourcerevision - this.gitrevision = gitrevision - this.remoteurl = remoteurl - this.remoteinfo = remoteinfo - } - - GitInfo(GitInfo another) { - super(another.script) - this.sourcepath = another.sourcepath - this.sourcerevision = another.sourcerevision - this.gitrevision = another.gitrevision - this.remoteurl = another.remoteurl - this.remoteinfo = another.remoteinfo - } - - void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { - this.sourcepath = sourcepath - this.sourcerevision = sourcerevision - this.gitrevision = gitrevision - this.remoteurl = remoteurl - this.remoteinfo = remoteinfo - } - - void set(GitInfo otherinfo) { - this.sourcepath = otherinfo.sourcepath - this.sourcerevision = otherinfo.sourcerevision - this.gitrevision = otherinfo.gitrevision - this.remoteurl = otherinfo.remoteurl - this.remoteinfo = otherinfo.remoteinfo - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - sourcepath = rawmeta['sourcepath'] - sourcerevision = rawmeta['sourcerevision'] - gitrevision = rawmeta['gitrevision'] - remoteurl = rawmeta['remoteurl'] - remoteinfo = rawmeta['remoteinfo'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'sourcepath' : sourcepath, - 'sourcerevision' : sourcerevision, - 'gitrevision' : gitrevision, - 'remoteurl' : remoteurl, - 'remoteinfo' : remoteinfo, - ] - return data - } -} - -class JenkinsBuildParameter extends GISJsonSerialised { - private String name - private String valuetype - private String value - - JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { - super(script) - this.name = name - this.valuetype = valuetype - this.value = value - } - - void set(String name, String valuetype, String value) { - this.name = name - this.valuetype = valuetype - this.value = value - } - - void deserialiseJson(rawmeta) { - name = rawmeta['name'] - valuetype = rawmeta['valuetype'] - value = rawmeta['value'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'name' : name, - 'valuetype' : valuetype, - 'value' : value, - ] - return data - } -} - -class JenkinsBuildInfo extends GISJsonSerialised { - private String format - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate - private ArrayList parameters - private boolean isfullbuild - - JenkinsBuildInfo(script, String buildjobname = '', int buildnumber = 0, String buildmachine = '', String builddate = '', boolean isfullbuild = true) { - super(script) - this.format = '0.1' - this.buildjobname = buildjobname - this.buildnumber = buildnumber - this.buildmachine = buildmachine - this.builddate = builddate - this.isfullbuild = isfullbuild - parameters = new ArrayList() - } - - void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { - this.buildjobname = buildjobname - this.buildnumber = buildnumber - this.buildmachine = buildmachine - this.builddate = builddate - this.isfullbuild = isfullbuild - } - - void set(boolean isfullbuild) { - this.isfullbuild = isfullbuild - } - - boolean isFullBuild() { - return isfullbuild - } - - void addParameter(String name, value) { - def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) - parameters.add(p) - return p - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - buildjobname = rawmeta['buildjobname'] - buildnumber = rawmeta['buildnumber'] - buildmachine = rawmeta['buildmachine'] - builddate = rawmeta['builddate'] - isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] - parameters = new ArrayList() - if (rawmeta['isfullbuild'] != null) { - for (rawparameter in rawmeta['parameters']) { - def p = new JenkinsBuildParameter(script) - p.deserialiseJson(rawparameter) - parameters.add(p) - } - } - } - - List parametersserialised() { - def associated = [] - for (p in parameters) { - associated.add(p.serialisationdata()) - } - return associated - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'buildjobname' : buildjobname, - 'buildnumber' : buildnumber, - 'buildmachine' : buildmachine, - 'builddate' : builddate, - 'isfullbuild' : isfullbuild, - 'parameters' : parametersserialised(), - ] - return data - } -} - -class BuildStampInfo extends GISJsonSerialised { - private String format - private String buildstampprefix - private int revision - private String buildstamp - - BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { - super(script) - this.format = '0.1' - this.buildstampprefix = buildstampprefix - this.revision = revision - this.buildstamp = buildstamp - } - - BuildStampInfo(BuildStampInfo another) { - super(another.script) - this.buildstampprefix = another.buildstampprefix - this.revision = another.revision - this.buildstamp = another.buildstamp - } - - void set(String buildstampprefix, int revision, String buildstamp) { - this.buildstampprefix = buildstampprefix - this.revision = revision - this.buildstamp = buildstamp - } - - void set(BuildStampInfo other) { - set(other.buildstampprefix, other.revision, other.buildstamp) - } - - // Override the buildstamp - void override(String buildstamp) { - this.buildstamp = buildstamp - script.echo("Build stamp overridden to: ${this.buildstamp}") - def s = this.buildstamp.split('\\.') - if (s.size() != 4) { - script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") - } - this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" - script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") - this.revision = s[3].toInteger() - script.echo("Revision overridden to: ${this.revision}") - } - - String buildstampcommasep() { - def String buildstampcommasep = buildstamp.replace(".", ",") - return buildstampcommasep - } - - void changerevision(int newrevision) { - script.echo("Build stamp currently ${buildstamp}") - revision = newrevision - def List buildstampsplit = buildstamp.split('\\.') - buildstampsplit[-1] = newrevision.toString() - buildstamp = buildstampsplit.join('.') - script.echo("Build stamp now ${buildstamp}") - } - - void set(String buildstampprefix, int revision) { - buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' - this.buildstampprefix = buildstampprefix - this.buildstamp = buildstampprefix + "0" - changerevision(revision) - } - - void set(String buildstampprefix, String revision = '0') { - set(buildstampprefix, revision.toInteger()) - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - buildstampprefix = rawmeta['buildstampprefix'] - revision = rawmeta['revision'] - buildstamp = rawmeta['buildstamp'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'buildstampprefix' : buildstampprefix, - 'revision' : revision, - 'buildstamp' : buildstamp, - ] - return data - } -} - -class ReleaseInfo extends GISJsonSerialised { - private String format - private String releaserepoURL - private ArrayList releaseitems - private boolean released - - ReleaseInfo(script, String releaserepoURL = '', boolean released = false) { - super(script) - this.format = '0.1' - this.releaserepoURL = releaserepoURL - this.releaseitems = new ArrayList() - this.released = released - } - - void set(String releaserepoURL = '') { - this.releaserepoURL = releaserepoURL - } - - void addReleaseItem(String releaseitem) { - script.echo("Adding release item: ${releaseitem} to release items") - releaseitems.add(releaseitem) - } - - void markreleased() { - this.released = true - } - - List releaseitemssserialised() { - def items = [] - for (ri in releaseitems) { - items.add(ri) - } - return items - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - releaserepoURL = rawmeta['releaserepoURL'] - releaseitems = new ArrayList() - for (rawreleaseitem in rawmeta['releaseitems']) { - releaseitems.add(rawreleaseitem) - } - released = rawmeta['released'] ?: false - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'releaserepoURL' : releaserepoURL, - 'releaseitems' : releaseitemssserialised(), - 'released' : released - ] - return data - } -} - -class ConfigurationInfo extends GISJsonSerialised { - private String configurationname - private String configurationURL - private String configurationversion - private String configurationSHA - - ConfigurationInfo(script, configurationname = '', configurationURL = '', configurationversion = '', configurationSHA = '') { - super(script) - this.configurationname = configurationname - this.configurationURL = configurationURL - this.configurationversion = configurationversion - this.configurationSHA = configurationSHA - } - - void deserialiseJson(rawmeta) { - configurationname = rawmeta['configurationname'] - configurationURL = rawmeta['configurationURL'] - configurationversion = rawmeta['configurationversion'] - configurationSHA = rawmeta['configurationSHA'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'configurationname' : configurationname, - 'configurationURL' : configurationURL, - 'configurationversion' : configurationversion, - 'configurationSHA' : configurationSHA, - ] - return data - } -} - -class ChecksumPair extends GISJsonSerialised { - private String filepath - private String sum - - ChecksumPair(script, String filepath = '', String sum = '') { - super(script) - this.filepath = filepath - this.sum = sum - } - - void deserialiseJson(rawmeta) { - filepath = rawmeta['filepath'] - sum = rawmeta['sum'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'filepath' : filepath, - 'sum' : sum, - ] - return data - } -} - -class ChecksumSet extends GISJsonSerialised { - private String format - private String artifactpath - private ArrayList checksumpairs - - ChecksumSet(script, artifactpath = '') { - super(script) - this.format = '0.1' - this.artifactpath = artifactpath - checksumpairs = new ArrayList() - } - - ChecksumPair addChecksumPair(String filepath, String sum) { - def cp = new ChecksumPair(script, filepath, sum) - checksumpairs.add(cp) - return cp - } - - List checksumpairsserialised() { - def associated = [] - for (cp in checksumpairs) { - associated.add(cp.serialisationdata()) - } - return associated - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - artifactpath = rawmeta['artifactpath'] - checksumpairs = new ArrayList() - for (rawchecksumpair in rawmeta['checksumpairs']) { - def csp = new ChecksumPair(script) - csp.deserialiseJson(rawchecksumpair) - checksumpairs.add(csp) - } - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'artifactpath' : artifactpath, - 'checksumpairs' : checksumpairsserialised(), - ] - return data - } -} - -class ChecksumInfo extends GISJsonSerialised { - private ArrayList checksumsets - - ChecksumInfo(script) { - super(script) - checksumsets = new ArrayList() - } - - ChecksumSet addChecksumSet(String artifactpath) { - def cs = new ChecksumSet(script, artifactpath) - checksumsets.add(cs) - return cs - } - - List checksumsetsserialised() { - def associated = [] - for (cs in checksumsets) { - associated.add(cs.serialisationdata()) - } - return associated - } - - void deserialiseJson(rawmeta) { - checksumsets = new ArrayList() - for (rawchecksumset in rawmeta['checksumsets']) { - def cs = new ChecksumSet(script) - cs.deserialiseJson(rawchecksumset) - checksumsets.add(cs) - } - } - - LinkedHashMap serialisationdata() { - def data = [ - 'checksumsets' : checksumsetsserialised(), - ] - return data - } -} - -class PublishableInfo extends GISJsonSerialised { - private String format - private String filepath - - PublishableInfo(script, String filepath = '') { - super(script) - this.format = '0.1' - this.filepath = filepath - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - filepath = rawmeta['filepath'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'filepath' : filepath, - ] - return data - } -} - -class MetaInfo extends GISJsonSerialised { - private static String METAFOLDERNAME = 'meta' - private static String METAFILENAME = 'info.json' - private static String METAARTIFACTS = "${METAFOLDERNAME}/**" - private static String format = '0.2' - - abstract class RelatedMetaInfo extends GISJsonSerialised { - String format - - RelatedMetaInfo(script, format) { - super(script) - this.format = format - } - - abstract void deserialiseJson(rawmeta) - abstract LinkedHashMap serialisationdata() - abstract String getrelatedid() - abstract void markreleased() - } - - class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { - private MetaInfo meta - private String offsetpath - private String uniqueid - - RelatedMetaInfo_v0_1(script, String offsetpath = '', MetaInfo meta = null, String uniqueid = UUID.randomUUID().toString()) { - super(script, '0.1') - this.offsetpath = offsetpath - this.uniqueid = uniqueid - this.meta = meta - } - - void deserialiseJson(rawmeta) { - offsetpath = rawmeta['offsetpath'] - uniqueid = rawmeta['uniqueid'] ?: uniqueid - meta = new MetaInfo(script) - meta.deserialiseJson(rawmeta['meta']) - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'offsetpath' : offsetpath, - 'uniqueid' : uniqueid, - 'meta' : meta.serialisationdata(), - ] - return data - } - - String getrelatedid() { - return uniqueid - } - - void markreleased() { - meta.releaseinfo.markreleased() - } - } - - class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { - private String importpath - private String offsetpath - private String uniqueid - - RelatedMetaInfo_v0_2(script, String importpath = '', String offsetpath = '', String uniqueid = UUID.randomUUID().toString()) { - super(script, '0.2') - this.importpath = importpath - this.offsetpath = offsetpath - this.uniqueid = uniqueid - } - - void deserialiseJson(rawmeta) { - //format = rawmeta['format'] - offsetpath = rawmeta['offsetpath'] - uniqueid = rawmeta['uniqueid'] - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'offsetpath' : offsetpath, - 'uniqueid' : uniqueid - ] - return data - } - - String getrelatedid() { - return uniqueid - } - - void markreleased() { - // Do nothing - } - } - - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo - private ChecksumInfo checksuminfo - private ArrayList configurationinfo - private ArrayList publishableinfo - private LinkedHashMap custominfo - private ArrayList associatedmeta - - MetaInfo(script) { - super(script) - productinfo = new ProductInfo(script) - gitinfo = new GitInfo(script) - jenkinsbuildinfo = new JenkinsBuildInfo(script) - buildstampinfo = new BuildStampInfo(script) - releaseinfo = new ReleaseInfo(script) - checksuminfo = new ChecksumInfo(script) - configurationinfo = new ArrayList() - publishableinfo = new ArrayList() - custominfo = new LinkedHashMap() - associatedmeta = new ArrayList() - } - - void markreleased() { - script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") - releaseinfo.markreleased() - for (associated in associatedmeta) { - associated.markreleased() - } - } - - List configurationinfoserialised() { - def configuration = [] - for (configurationitem in configurationinfo) { - configuration.add(configurationitem.serialisationdata()) - } - return configuration - } - - void addPublishable(String filepath) { - def PublishableInfo pi = new PublishableInfo(script, filepath) - script.echo("Adding ${filepath} to publishable list") - publishableinfo.add(pi) - } - - void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { - def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) - script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") - configurationinfo.add(ci) - return ci - } - - void addCustomInfo(String customId, String customValue) { - custominfo.put(customId.trim(), customValue) - } - - String getCustomInfo(String customId) { - return custominfo.getAt(customId.trim()) - } - - def isFullBuild() { - return jenkinsbuildinfo.isFullBuild() - } - - List publishableinfoserialised() { - def publishable = [] - for (publishableitem in publishableinfo) { - publishable.add(publishableitem.serialisationdata()) - } - return publishable - } - - List associatedmetaserialised() { - def associated = [] - for (metaitem in associatedmeta) { - associated.add(metaitem.serialisationdata()) - } - return associated - } - - void deserialiseJson(rawmeta) { - if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { - script.error("Unsupported meta format found") - return - } - productinfo.deserialiseJson(rawmeta['productinfo']) - gitinfo.deserialiseJson(rawmeta['gitinfo']) - jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) - buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) - releaseinfo.deserialiseJson(rawmeta['releaseinfo']) - checksuminfo.deserialiseJson(rawmeta['checksuminfo']) - for (rawconfig in rawmeta['configuration']) { - def cf = new ConfigurationInfo(script) - cf.deserialiseJson(rawconfig) - configurationinfo.add(cf) - } - for (rawpublishable in rawmeta['publishableinfo']) { - def pb = new PublishableInfo(script) - pb.deserialiseJson(rawpublishable) - publishableinfo.add(pb) - } - custominfo = rawmeta['custominfo'] - for (rawrelatedmeta in rawmeta['associatedmeta']) { - if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { - def rm = new RelatedMetaInfo_v0_1(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) - } else if (rawrelatedmeta['format'] == '0.2') { - def rm = new RelatedMetaInfo_v0_2(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) - } - } - } - - def deserialiseJson(String filepath) { - script.echo("Deserialising JSON at file: ${filepath}") - def rawmeta = script.readJSON(file: filepath) - deserialiseJson(rawmeta) - } - - def deserialiseJsonSlurper(String filepath) { - File jsonfile = new File(filepath) - def jsonfiletext = jsonfile.getText() - def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) - deserialiseJson(rawmeta) - } - - String relativize(String mypath) { - String pathRelative - //script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") - if (mypath.startsWith(script.WORKSPACE)) { - pathRelative = mypath.minus(script.WORKSPACE) - //script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") - } else { - pathRelative = mypath - //script.echo("Path relative is unchanged: ${pathRelative}") - } - // Convert \ with / - pathRelative = pathRelative.tokenize('\\').join('/') - // Strip leading / if any - pathRelative = pathRelative.replaceAll('^/+', '') - - //script.echo("Path relative is ${pathRelative}") - - return pathRelative - } - - static String metafilepath() { - return METAFOLDERNAME + "/" + METAFILENAME - } - - MetaInfo importrelated(boolean appendtoparentmeta, String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { - MetaInfo mi = new MetaInfo(script) - - def String adjustedpath = relativize(destpath) - def importedmetafile = adjustedpath + "/" + metafilepath() - if (script.fileExists(importedmetafile)) { - //script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") - mi.deserialiseJson(importedmetafile) - - RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) - if (appendtoparentmeta) { - associatedmeta.add(rmi) - } - } else { - script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") - addConfigurationInfo('', givenRepo, givenRevision, givenSHA) - } - return mi - } - - def copyrelatedmeta() { - script.echo("Copying related input meta data to output meta") - for (relatedmetainfo in associatedmeta) { - def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" - def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" - script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") - script.fileOperations([ - script.folderCreateOperation(folderPath: destpath), - script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) - ]) - } - } - - LinkedHashMap serialisationdata() { - def data = [ - 'format' : format, - 'productinfo' : productinfo.serialisationdata(), - 'gitinfo' : gitinfo.serialisationdata(), - 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), - 'buildstampinfo' : buildstampinfo.serialisationdata(), - 'releaseinfo' : releaseinfo.serialisationdata(), - 'configuration' : configurationinfoserialised(), - 'checksuminfo' : checksuminfo.serialisationdata(), - 'publishableinfo' : publishableinfoserialised(), - 'custominfo' : custominfo, - 'associatedmeta' : associatedmetaserialised(), - ] - return data - } - - static String folder() { - return METAFOLDERNAME - } - - def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here - ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here - String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" - ) { - def String artifactlist = METAARTIFACTS - releaseinfo.addReleaseItem(METAARTIFACTS) - for (releaseitem in releasedfromhere) { - artifactlist += "," + releaseitem - releaseinfo.addReleaseItem(releaseitem) - } - for (releaseitem in releasedfromsubmeta) { - artifactlist += "," + releaseitem - } - saveJsonSerialised(metafilepath) - return artifactlist - } - - String componentidentifier() { - return productinfo.productname + "_" + productinfo.subcomponentname - } - - String productvariant() { - if (!productinfo.variant) { - return productinfo.productname - } else { - return productinfo.productname + "_" + productinfo.variant - } - } - - String componentversionuniqueidentifier() { - return componentidentifier() + "_" + buildstampinfo.buildstamp - } - - String globalreleasetag() { - return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber - } - - String releasetag() { - return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber - } - - String releasetagfordebpackage() { - return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber - } - - class CollectedMeta { - private MetaInfo metainfo - private String accumulatedpath - private String accumulatedmetapath - - CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { - this.metainfo = metainfo - this.accumulatedpath = accumulatedpath - this.accumulatedmetapath = accumulatedmetapath - } - } - - class IterationInfo { - private ArrayList metacollection - private Map metamap - private String comment - private int level - private String gatheredpath - private String gatheredmetapath - private boolean limiteddepth - private int maxdepth - - IterationInfo(ArrayList metacollection, - Map metamap, - String comment = '', - int level = 0, - String gatheredpath = '', - String gatheredmetapath = '', - boolean limiteddepth = false, - int maxdepth = 0) { - this.metacollection = metacollection - this.metamap = metamap - this.comment = comment - this.level = level - this.gatheredpath = gatheredpath - this.gatheredmetapath = gatheredmetapath - this.limiteddepth = limiteddepth - this.maxdepth = maxdepth - } - - String show() { - return "Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}" - } - - void add(CollectedMeta item) { - metacollection.add(item) - metamap.put(item.metainfo.productinfo.uniqueid, item) - } - } - - def openchildmeta(IterationInfo ii, RelatedMetaInfo relatedmetainfo) { - def MetaInfo mi = new MetaInfo(script) - - def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") - def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" - def boolean imported - // If this has already been read in from a file - if (relatedmetainfo.getrelatedid() in ii.metamap) { - //script.echo("Already read in meta associated with uniqueid: ${relatedmetainfo.getrelatedid()} - Using it") - mi = ii.metamap[relatedmetainfo.getrelatedid()].metainfo - imported = true - } else if (script.fileExists(importedmetafile)) { - mi.deserialiseJson(importedmetafile) - imported = true - } else { - script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") - imported = false - } - - def childmetainfo = [ - 'imported': imported, - 'nextgatheredpath': nextgatheredpath, - 'nextgatheredmetapath': nextgatheredmetapath, - 'childmeta': mi - ] - return childmetainfo - } - - def walkrelated_v0_1(IterationInfo ii) { - //script.echo("Walk related v0.1: ${ii.show()}") - if (!ii.limiteddepth || ii.maxdepth > 0) { - ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) - ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { - def MetaInfo othermeta = relatedmetainfo.meta - def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") - def nextii = new IterationInfo(ii.metacollection, - ii.metamap, - ii.comment, - ii.level + 1, - nextgatheredpath, - nextgatheredmetapath, - ii.limiteddepth, - ii.maxdepth - 1) - def results = othermeta.walkrelated_v0_1(nextii) - ii.metacollection = results.metacollection - ii.metamap = results.metamap - ii.comment = results.comment - } else { - script.error("There is non v0.1 format related meta info under v0.1 related meta info") - } - } - } - return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] - } - - def walkrelated(IterationInfo ii) { - //script.echo("Walk related: ${ii.show()}") - if (!ii.limiteddepth || ii.maxdepth > 0) { - ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) - ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - //script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") - if (relatedmetainfo.format == '0.2') { - def childmetainfo = openchildmeta(ii, relatedmetainfo) - if (childmetainfo.imported) { - def nextii = new IterationInfo(ii.metacollection, - ii.metamap, - ii.comment, - ii.level + 1, - childmetainfo.nextgatheredpath, - childmetainfo.nextgatheredmetapath, - ii.limiteddepth, - ii.maxdepth - 1) - def results = childmetainfo.childmeta.walkrelated(nextii) - ii.metacollection = results.metacollection - ii.metamap = results.metamap - ii.comment = results.comment - } - } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { - def MetaInfo othermeta = relatedmetainfo.meta - def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") - def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") - def nextii = new IterationInfo(ii.metacollection, - ii.metamap, - ii.comment, - ii.level + 1, - nextgatheredpath, - nextgatheredmetapath, - ii.limiteddepth, - ii.maxdepth - 1) - def results = othermeta.walkrelated_v0_1(nextii) - ii.metacollection = results.metacollection - ii.metamap = results.metamap - ii.comment = results.comment - } - } - } - return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] - } - - // Walks all layers of associated data only - def allrelatedmeta(String metapath = '') { - def ArrayList metacollection = new ArrayList() - def Map metamap = new LinkedHashMap() - return walkrelated(new IterationInfo(metacollection, metamap, '', 0, '', metapath)) - } - - ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = new ArrayList() - def Map metamap = new LinkedHashMap() - def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) - //script.echo("Release walker iteration info at start:\n${ii.show()}") - def results = walkrelated(ii) - return results.metacollection - } - - // Walks 1 layer of associated data only - def getassociated(String rootpath) { - return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself) - } - - Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { - if (releasemetacollection.containsKey(uniquestring)) { - def existingentry = releasemetacollection[uniquestring] - def existingentrypathlength = existingentry.accumulatedpath.split('/').size() - def newentrypathlength = metaitem.accumulatedpath.split('/').size() - if (existingentrypathlength > newentrypathlength) { - releasemetacollection.put(uniquestring, metaitem) - } - } else { - releasemetacollection.put(uniquestring, metaitem) - } - return releasemetacollection - } - - Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - - def Map versionuniquecollection = new LinkedHashMap() - for (metaitem in metacollection) { - def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() - versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) - } - - return versionuniquecollection - } - - Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { - def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - - def Map uniquecollection = new LinkedHashMap() - for (metaitem in metacollection) { - def String uniquestring = metaitem.metainfo.componentidentifier() - uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) - } - - return uniquecollection - } - - MetaInfo getNewMetaObject(script) { - return new MetaInfo(script) - } - - def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { - def results = [ - 'matchedmetaitem' : new CollectedMeta(this), - 'matchfound' : false - ] - def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") - for (dependentmetaitem in walkresults.metacollection) { - script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") - if (dependentmetaitem.metainfo.productinfo.productname == productname && - dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { - script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") - results.matchedmetaitem = dependentmetaitem - results.matchfound = true - break - } - } - return results - } -} - -MetaInfo metareset(script) { - def mi = new MetaInfo(script) - - echo "Resetting meta folder" - dir(mi.folder()) { - deleteDir() - } - - return mi -} - -MetaInfo metafromfolder(String folder = '') { - MetaInfo buildmeta = new MetaInfo(this) - buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") - return buildmeta -} - -MetaInfo metafromfolderSlurper(String folder = '') { - MetaInfo buildmeta = new MetaInfo(this) - buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") - return buildmeta -} - -return this; diff --git a/Common_scripts/packaging/debian.groovy b/Common_scripts/packaging/debian.groovy deleted file mode 100644 index 9f2f4d3..0000000 --- a/Common_scripts/packaging/debian.groovy +++ /dev/null @@ -1,111 +0,0 @@ -class DebRootDefinition { - String ROOTPATH - String PACKAGE_PREFIX - String PACKAGE_FOLDER_NAME - String PACKAGE_NAME - String UNIQUE_PACKAGE_NAME - - String PACKAGE_SOURCE_ROOT - String PACKAGE_SOURCE_FOLDER_NAME - String DEBPATH - - String ETC - String GIS_OPT_BIN - String GIS_OPT_LIB - String GIS_OPT_SCRIPTS_APP - String GIS_OPT_SCRIPTS_EL - String GIS_OPT_SCRIPTS_CONTROL - String GIS_OPT_WWW - String GIS_OPT_OTHER - String GIS_OPT_KERNEL - String GIS_OPT_LOGIC - String GIS_OPT_PRINTHEAD - String GIS_OPT_BOOT - - String USR_BIN - String USR_LIB_ARCH - String LIB_FIRMWARE - String HTTP_DEV_DIV - String LIB_SYSTEMD - - DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix) { - ROOTPATH = rootpath - PACKAGE_PREFIX = packageprefix - PACKAGE_FOLDER_NAME = packageprefix - PACKAGE_NAME = packageprefix + "_armhf.deb" - UNIQUE_PACKAGE_NAME = packageprefix + "_" + buildstamp + "_armhf.deb" - - PACKAGE_SOURCE_ROOT = packagesourceroot - PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" - DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" - - ETC = "${DEBPATH}/etc" + inpackagepathsuffix - GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + inpackagepathsuffix - GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + inpackagepathsuffix - GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + inpackagepathsuffix - GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + inpackagepathsuffix - GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" + inpackagepathsuffix - GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + inpackagepathsuffix - GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + inpackagepathsuffix - GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" + inpackagepathsuffix - GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + inpackagepathsuffix - GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + inpackagepathsuffix - GIS_OPT_BOOT = "${DEBPATH}/opt/gis/boot" + inpackagepathsuffix - - USR_BIN = "${DEBPATH}/usr/bin" + inpackagepathsuffix - USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + inpackagepathsuffix - LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + inpackagepathsuffix - HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + inpackagepathsuffix - LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + inpackagepathsuffix - } -} - -def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix = '') { - return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) -} - -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { - withCredentials([usernamePassword(credentialsId: credentials_id, - usernameVariable: 'SUDO_USERNAME', - passwordVariable: 'SUDO_PASSWORD')]) { - def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" - def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" - try { - sh """#!/bin/bash - # Exit if any command fails - set -xe - - cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - - ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" - - mkdir -v -p "${packageout}" - echo "Building package:" - dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" - - mkdir -v -p "${uniquepackageout}" - cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" - - echo "Showing package:" - dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package contents:" - dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - - echo "Showing package lintian output:" - lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - - ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - """ - } - catch (Exception e) { - echo "Exception occurred" - sh """#!/bin/bash -v - ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - - exit 1 - """ - } - } -} - -return this; \ No newline at end of file diff --git a/Common_scripts/platform/lineendings.groovy b/Common_scripts/platform/lineendings.groovy deleted file mode 100644 index 9657b0d..0000000 --- a/Common_scripts/platform/lineendings.groovy +++ /dev/null @@ -1,5 +0,0 @@ - -// Put ${nl} into strings when wanting a newline -nl = System.properties['line.separator'] - -return this; diff --git a/Common_scripts/product/info.groovy b/Common_scripts/product/info.groovy deleted file mode 100644 index 67d1449..0000000 --- a/Common_scripts/product/info.groovy +++ /dev/null @@ -1,302 +0,0 @@ - -importer = load("${autobuilder.commonscripts()}/import/importer.groovy") -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -rt = load("${autobuilder.commonscripts()}/release/tag.groovy") - -class ProductDependency { - private script - private rtscript - private String productvariant - private String releaserepo - private String presplitreleasetag - private String releasetag - private String releasestate - - def ProductDependency(script, - rtscript, - String productvariant, - String presplitreleasetag, - String releaserepo) { - this.script = script - this.rtscript = rtscript - this.productvariant = productvariant - this.releaserepo = releaserepo - this.presplitreleasetag = presplitreleasetag - this.releasetag = '' - this.releasestate = '' - } - - def setreleasestate() { - def LinkedHashMap results = rtscript.startswithspecialprefix(presplitreleasetag) - if (results.hasspecialprefix) { - releasetag = results.tagwithoutprefix - releasestate = results.prefixfound - } else { - releasetag = results.tagwithoutprefix - releasestate = '' - } - } -} - -class ProductComponent { - private String componentname - private String releaserepo - private String releasetag - - def ProductComponent(String componentname, String releasetag, String releaserepo) { - this.componentname = componentname - this.releaserepo = releaserepo - this.releasetag = releasetag - } -} - -class ProductRelease { - private String sha - private String builddate - private ArrayList tags - private ArrayList productdependencies - private ArrayList productcomponents - - def ProductRelease(String sha, String builddate, ArrayList tags) { - this.sha = sha - this.builddate = builddate - this.tags = tags - this.productdependencies = new ArrayList() - this.productcomponents = new ArrayList() - } - - def addProductDependency(ProductDependency dependency) { - productdependencies.add(dependency) - } - - def addProductComponent(ProductComponent component) { - productcomponents.add(component) - } -} - -class ProductParsed { - enum State { - UNPARSED, - PARSED, - } - private State scanstate - private String productvariant - private ArrayList releases - - def ProductParsed(String productvariant) { - scanstate = State.UNPARSED - this.productvariant = productvariant - this.releases = new ArrayList() - } - - def markparsed() { - scanstate = State.PARSED - } - - def markparseerror() { - scanstate = State.PARSED - } - - def boolean parsed() { - return (scanstate == State.PARSED) - } - - def addProductRelease(ProductRelease productrelease) { - releases.add(productrelease) - } -} - -def String translateproductvariant(productvarianttranslation, - String releaserepo, - String productvariant) { - def translatedproductvariant = productvarianttranslation.get(releaserepo) - if (translatedproductvariant != null) { - echo "Translating product variant: ${productvariant} to: ${translatedproductvariant}" - return translatedproductvariant - } - return productvariant -} - -def ProductParsed addProductEntry(LinkedHashMap productcollection, - productvarianttranslation, - String productvariant, - String releaserepo) { - def ProductParsed productparsed - - if (productvariant && releaserepo) { - if (productcollection.get(releaserepo) == null) { // Use the first entry name only - echo "Adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" - def translatedproductvariant = translateproductvariant(productvarianttranslation, - releaserepo, - productvariant) - productparsed = new ProductParsed(translatedproductvariant) - productcollection.put(releaserepo, productparsed) - } else { - echo "Already added product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" - } - productparsed = productcollection.get(releaserepo) - } else { - echo "Not adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" - } - return productparsed -} - -def parseproductdependencies(LinkedHashMap productcollection, - productvarianttranslation, - productreleasemeta, - String productreleasefolder, - String containingproductvariant, - ProductRelease productrelease) { - def dependentmeta = productreleasemeta.getassociated(productreleasefolder) - - for (meta in dependentmeta) { - def translatedproductvariant = translateproductvariant(productvarianttranslation, - meta.releaseinfo.releaserepoURL, - meta.productvariant()) - if (translatedproductvariant && meta.productinfo.subcomponentname) { - def pd = new ProductDependency(this, - rt, - translatedproductvariant, - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - pd.setreleasestate() - if (meta.productinfo.subcomponentname == 'WHOLE' && translatedproductvariant) { - if (translatedproductvariant != containingproductvariant) { // Don't add myself - addProductEntry(productcollection, - productvarianttranslation, - translatedproductvariant, - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) - } - } else { - def pc = new ProductComponent(meta.productinfo.subcomponentname, - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductComponent(pc) - } - } else { - echo "Ignoring unnamed dependency in ${productreleasemeta.productvariant()}" - } - } -} - -def parseproductrepo(LinkedHashMap productcollection, - productvarianttranslation, - String productrepo, - String productreleasefolder) { - def boolean productrepogood = true - echo "Querying repo: ${productrepo}" - dir(productreleasefolder) { - deleteDir() - def returnstatus = sh(script: """#!/bin/bash - git init - if [ \$? -ne 0 ]; then exit 1; fi - git config remote.origin.url ${productrepo} - if [ \$? -ne 0 ]; then exit 1; fi - git config advice.detachedHead false - if [ \$? -ne 0 ]; then exit 1; fi - git sparse-checkout init - if [ \$? -ne 0 ]; then exit 1; fi - git sparse-checkout set meta/* - if [ \$? -ne 0 ]; then exit 1; fi - git pull -t - if [ \$? -ne 0 ]; then exit 1; fi - """, - returnStatus: true) - if (returnstatus > 0) { - dir('meta') { - deleteDir() - } - echo "Couldn't clone repository: ${productrepo} correctly" - productrepogood = false - } - } - if (productrepogood) { - def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) - def String productvariant - def ProductParsed pp - shatagmap.each { sha, tags -> - //echo "Sha: ${sha} has tags: ${tags}" - dir(productreleasefolder) { - def returnstatus = sh(script: """#!/bin/bash - git checkout ${sha} - """, - returnStatus: true) - if (returnstatus > 0) { - dir('meta') { - deleteDir() - } - } - } - def productreleasemeta = meta.importrelated(false, productreleasefolder) - if (!productvariant) { - productvariant = productreleasemeta.productvariant() - pp = addProductEntry(productcollection, - productvarianttranslation, - productvariant, - productrepo) - if (pp) { - echo "Product variant created for: ${productrepo} name: ${pp.productvariant}" - } else { - echo "No product entry added/associated with ${productrepo} name: ${productvariant}" - } - } - - if (pp) { - def productrelease = new ProductRelease(sha, - productreleasemeta.jenkinsbuildinfo.builddate, - tags) - echo "Productrelease: ${productrelease.sha} created" - parseproductdependencies(productcollection, - productvarianttranslation, - productreleasemeta, - productreleasefolder, - pp.productvariant, - productrelease) - echo "Productrelease: ${productrelease.sha} added to Productvariant: ${pp.productvariant}" - pp.addProductRelease(productrelease) - } - } - if (pp) { - pp.markparsed() - echo "Productvariant: ${pp.productvariant} parsed" - } - } -} - -def iterateinitialproductrepos(initialproductcollection, - productvarianttranslation, - LinkedHashMap productcollection, - String productreleasefolder) { - for (productrepo in initialproductcollection.ProductReleaseRepos) { - parseproductrepo(productcollection, - productvarianttranslation, - productrepo, - productreleasefolder) - } - while( true ) { - boolean allparsed = true - echo "Parsing extra products" - productcollection.each { releaserepo, productparsed -> - if (!productparsed.parsed()) { - allparsed = false - parseproductrepo(productcollection, - productvarianttranslation, - releaserepo, - productreleasefolder) - } else { - echo "Parsed already ${productparsed.productvariant}" - } - } - if (allparsed) { - echo "All parsed" - break - } - } -} - -def LinkedHashMap getNewProductCollection() { - return new LinkedHashMap() -} - -return this \ No newline at end of file diff --git a/Common_scripts/publish/.classpath b/Common_scripts/publish/.classpath deleted file mode 100644 index 287c1ab..0000000 --- a/Common_scripts/publish/.classpath +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/Common_scripts/publish/.gitignore b/Common_scripts/publish/.gitignore deleted file mode 100644 index abed335..0000000 --- a/Common_scripts/publish/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -/do$_publish_closure4.class -/do$_resetfolders_closure1.class -/do$_resetfolders_closure2.class -/do$_resetfolders_closure3.class -/do$_stripgithistory_closure5.class -/do.class -/publish_item_select.class diff --git a/Common_scripts/publish/.project b/Common_scripts/publish/.project deleted file mode 100644 index 5c60328..0000000 --- a/Common_scripts/publish/.project +++ /dev/null @@ -1,18 +0,0 @@ - - - publish_item_select - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.groovy.core.groovyNature - org.eclipse.jdt.core.javanature - - diff --git a/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs deleted file mode 100644 index 1e68ed6..0000000 --- a/Common_scripts/publish/.settings/org.eclipse.jdt.groovy.core.prefs +++ /dev/null @@ -1,3 +0,0 @@ -eclipse.preferences.version=1 -groovy.compiler.level=25 -groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/Common_scripts/publish/do.groovy b/Common_scripts/publish/do.groovy deleted file mode 100644 index 8e302d8..0000000 --- a/Common_scripts/publish/do.groovy +++ /dev/null @@ -1,174 +0,0 @@ -import groovy.io.FileType - -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" -releaserepofolder = "${WORKSPACE}/releaserepos" -filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(filteredreleaserepofolder) { - deleteDir() - } -} - -def productuniquefolder(releasemeta) { - return "${releasemeta.productvariant()}_${releasemeta.productinfo.subcomponentname}" -} - -def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { - return "Software/${productline} ${softwarerelease}/${productuniquefolder(releasemeta)}" -} - -def uniqueabsoluterreleasefolder(String productline, String releaserepofolder, String softwarerelease, releasemeta) { - return "${releaserepofolder}/${uniquereleasefolder(productline, softwarerelease, releasemeta)}" -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Publish summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason = reason -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def guessreporootsuffix(componentURL) { - def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - return guess -} - -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { - summaryappend("Artifacts: ") - - def releasemeta = releaseinfo.metainfo - def String componentURL = releasemeta.releaseinfo.releaserepoURL - def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" - def publishableitems = releasemeta.publishableinfo - - def ArrayList publishablepaths = new ArrayList() - if (publishableitems.size() > 0) { - for (publishableitem in publishableitems) { - def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" - echo "publishableitem: ${productuniquepublishableitem}" - if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { - publishablepaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") - } - } - } - if (publishablepaths.size() > 0) { - def reporootsuffix = guessreporootsuffix(componentURL) - gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone - } - else { - summaryappend("None ") - } -} - -def publish(hostedftpsiteinfo, String customer, String finalreleaserepofolder = releaserepofolder) { - summaryappend(" FTP operations: ") - - def String ftproot = "${hostedftpsiteinfo.workingdirectory}${customer}" - dir(finalreleaserepofolder) { - summaryappend("FTP operations: Transferring to ${ftproot}") - sh """#!/bin/bash - ncftpput -u ${hostedftpsiteinfo.username} -p ${hostedftpsiteinfo.password} -R ${hostedftpsiteinfo.url} "${ftproot}" . - """ - summaryappend(", Transferred ") - } -} - -def copydirectory(File dirFrom, File dirTo) { - if (!dirTo.exists()) { - dirTo.mkdirs(); - } - dirFrom.eachFile(FileType.FILES) { File source -> - File target = new File(dirTo, source.getName()) - target.bytes = source.bytes - } - dirFrom.eachFile(FileType.DIRECTORIES) { File source -> - File target = new File(dirTo, source.getName()) - copyLargeDir(source, target) - } -} - -def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { - def releasemeta = releaseinfo.metainfo - def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) - - summaryappend("${releasemeta.productvariant()} ${releasemeta.productinfo.subcomponentname}: ") - - getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) - - summaryappend("\n") -} - -def convertitemstopublish(String itemstopublishstring) { - def List paths = [] - - def String[] items = itemstopublishstring.split(',') - def String allpaths = "" - for(String item : items) { - def String[] pathitems = item.split('#') - def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" - paths.add(path) - allpaths += "${path}\n" - } - echo "Selected item to publish:\n${allpaths}" - return paths -} - -// Publish selected items from a release to the customer folder on the ftp site -// itemstopublishstring is a comma-seperated set of strings in this format ## -def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { - int md = maxdepth.toInteger() - resetfolders() - preconditions() - - def productline = releasemeta.productvariant() - // Walk the meta information to a max depth, looking for artifacts to publish - def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) - - echo "Version unique components:" - def String versionuniquecomponent = "" - for (key in componentReleaseInfoVersionUnique.keySet()) { - versionuniquecomponent += "${key}\n" - } - echo versionuniquecomponent - - def itempathstopublish = convertitemstopublish(itemstopublishstring) - for (key in componentReleaseInfoVersionUnique.keySet()) { - def releaseinfo = componentReleaseInfoVersionUnique[key] - recallcomponent(productline, releaseinfo, softwarerelease, itempathstopublish) - } - gh.stripgithistory(releaserepofolder) - publish(hostedftpsiteinfo, customer) - - return [summarystring, markunstable, unstablereason] -} - -def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String customer, String softwarerelease, String maxdepth) { - return publishhierarchy(hostedftpsiteinfo, releasemeta, releasemetapath, '', customer, softwarerelease, maxdepth) -} - -return this; diff --git a/Common_scripts/publish/package.groovy b/Common_scripts/publish/package.groovy deleted file mode 100644 index 04236e0..0000000 --- a/Common_scripts/publish/package.groovy +++ /dev/null @@ -1,165 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - PACKAGEDEST = "${WORKSPACE}/package" - cleanfolders = [ - PACKAGEDEST, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Copying package artifacts') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - def packageimportoptions = importer.getNewImportOptions() - // For backward compatibility - in case these parameters are not supplied - def packagelocation = params['PACKAGE_LOCATION'] ?: 'Jenkins job artifact' - def packagereleaserepo = params['PACKAGE_RELEASE_REPO'] ?: '' - def packagereleaserevision = params['PACKAGE_RELEASE_REVISION'] ?: '' - packageimportoptions.source(packagelocation, - PACKAGE_CREATE_JOB, - PACKAGE_CREATE_JOB_SELECTOR, - packagereleaserepo, - packagereleaserevision) - packageimportoptions.setincrementversionnumber(false) - packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) - - PACKAGE_FOLDER = "${PACKAGEDEST}/package" - UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" - } - } - } - stage('Aptly environment') { - steps { - script { - echo "Aptly config file is ${APTLY_CONFIG_FILE}" - sh """#!/bin/bash -v - aptly config show - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Create repo - if not already there') { - steps { - script { - echo "Repo name is ${REPO_NAME}" - sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" - if [ \$? -ne 0 ]; then - echo "aptly repo ${REPO_NAME} does not exist; Creating it" - aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - fi - aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Drop published repo - if any') { - steps { - script { - sh """#!/bin/bash -v - # This errors if the published repo exists already; Ignore it - aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" - aptly publish -config="${APTLY_CONFIG_FILE}" list - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - - stage('Add/update package in repo') { - steps { - script { - echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" - sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" - if [ \$? -ne 0 ]; then exit 1; fi - aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Create snapshot') { - steps { - script { - echo "Snapshot name is ${SNAPSHOT_NAME}" - FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" - PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix - echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" - // Convert prefix to regular expression - PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") - PACKAGE_VERSION_REGEX = '' - i = 0 - while(true) { - if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { - PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] - break; - } - PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] - PACKAGE_VERSION_REGEX += '\\.' - i++ - } - sh """#!/bin/bash -v - # This errors if the snapshot exists already; Ignore it - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - - aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' - if [ \$? -ne 0 ]; then exit 1; fi - aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - stage('Publish snapshot') { - steps { - script { - sh """#!/bin/bash -v - aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" - if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" list - if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/packagecollection.groovy b/Common_scripts/publish/packagecollection.groovy deleted file mode 100644 index a87453e..0000000 --- a/Common_scripts/publish/packagecollection.groovy +++ /dev/null @@ -1,183 +0,0 @@ -def packagecollection = null - -/* Example expected JSON -{ -{ - "packages": [ - { - "Name": "gis-hmb-fd-hv-esw-el-sg1024", - "Location": "Jenkins job artifact", - "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_app_0.9.6_debpackage", - "Job selector": "1248", - "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_package.git", - "Release repo revision": "refs/tag/0.9.6.57669", - "Path": "package/gis-hmb-fd-hv-esw-el-sg1024_armhf.deb" - }, - { - "Name": "gis-hmb-fd-hv-control-service-sg1024", - "Location": "Jenkins job artifact", - "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_control_0.9.6_debpackage", - "Job selector": "1722", - "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_control_package.git", - "Release repo revision": "refs/tag/0.9.6.7964", - "Path": "package/gis-hmb-fd-hv-control-service-sg1024_armhf.deb" - } - ] -} -*/ - -def String asJsonText(somejson = packagecollection) { - return writeJSON(returnText: true, json: somejson, pretty: 1) -} - -def boolean packagecollection_exists() { - return (packagecollection != null) -} - -def packagecollection_set(String jsontext) { - packagecollection = readJSON(text: jsontext) - echo "Package collection info:\n${asJsonText(packagecollection)}\nType:\n${packagecollection.getClass()}" -} - -def packagecollection_get() { - return packagecollection -} - -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - PACKAGEDEST = "${WORKSPACE}/package" - cleanfolders = [ - PACKAGEDEST, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Transfer packages') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - packagecollection_set(PACKAGE_COLLECTION) - debpackagecollection = packagecollection_get() - - debpackagecollection.packages.each { debpackageinfo-> - echo "Processing package: ${debpackageinfo.Name}\n${debpackageinfo}\n${debpackageinfo.getClass()}" - echo "Info: ${debpackageinfo.Location}" - echo "Info: ${debpackageinfo['Location']}" - echo "Info: ${debpackageinfo['Job']}" - echo "Info: ${debpackageinfo['Job selector']}" - echo "Info: ${debpackageinfo['Release repo']}" - echo "Info: ${debpackageinfo['Release repo revision']}" - - def packagedest = "${PACKAGEDEST}/${debpackageinfo.Name}" - - def packageimportoptions = importer.getNewImportOptions() - // For backward compatibility - in case these parameters are not supplied - packageimportoptions.source(debpackageinfo['Location'], - debpackageinfo['Job'], - "${debpackageinfo['Job selector']}", - debpackageinfo['Release repo'], - debpackageinfo['Release repo revision']) - packageimportoptions.setincrementversionnumber(false) - packagemeta = importer.artifactimport(meta, packageimportoptions, packagedest) - - UNIQUE_PACKAGE_FOLDER = "${packagedest}/package" - - echo "Aptly config file is ${APTLY_CONFIG_FILE}" - echo "Repo name is ${REPO_NAME}" - echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" - - sh """#!/bin/bash -v - aptly config show - if [ \$? -ne 0 ]; then exit 1; fi - - aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" - if [ \$? -ne 0 ]; then - echo "aptly repo ${REPO_NAME} does not exist; Creating it" - aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - fi - aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - - # This errors if the published repo exists already; Ignore it - aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" - aptly publish -config="${APTLY_CONFIG_FILE}" list - if [ \$? -ne 0 ]; then exit 1; fi - - aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" - if [ \$? -ne 0 ]; then exit 1; fi - aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - - echo "Snapshot name is ${SNAPSHOT_NAME}" - FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" - PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix - echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" - // Convert prefix to regular expression - PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") - PACKAGE_VERSION_REGEX = '' - i = 0 - while(true) { - if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { - PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] - break; - } - PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] - PACKAGE_VERSION_REGEX += '\\.' - i++ - } - sh """#!/bin/bash -v - # This errors if the snapshot exists already; Ignore it - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - - aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' - if [ \$? -ne 0 ]; then exit 1; fi - aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" - if [ \$? -ne 0 ]; then exit 1; fi - - aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" - if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" list - if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" - if [ \$? -ne 0 ]; then exit 1; fi - """ - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/promptingpublish.groovy b/Common_scripts/publish/promptingpublish.groovy deleted file mode 100644 index 49d9421..0000000 --- a/Common_scripts/publish/promptingpublish.groovy +++ /dev/null @@ -1,184 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean folders') { - steps { - script { - cleanfolders = [ - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Checkout release repo') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - SOURCEURL = RELEASE_REPO - SOURCEREVISION = "origin/master" - SOURCEDIR = "${WORKSPACE}/whole" - configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - } - } - } - stage('Identify customer to publish to') { - steps { - script { - hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") - def List customers = hostedftb.getCustomerList() - timeout(time: 10, unit: 'MINUTES') { - customer = input( - message: 'Please select customer to publish to', - ok: 'Ok', - parameters: [ - choice(name: 'customer', choices: customers, description: 'Which customer?') - ] - ) - echo "Chosen customer is: ${customer}" - } - } - } - } - stage('Identify software release to publish') { - steps { - script { - rt = load("${autobuilder.commonscripts()}/release/tag.groovy") - gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def allreleases = gh.gettaglist(SOURCEDIR) - def releases = new ArrayList() - for (release in allreleases) { - def boolean canadd = true - if (FILTER_REDUNDANT_RELEASES == 'true') { - def results = rt.startswithspecialprefix(release) - if (results.prefixfound) { - canadd = false - } - } - if (FILTER_LONGNAME_RELEASES == 'true') { - if (release.findAll(/[a-zA-Z]/)) { - canadd = false - } - } - if (canadd) { - echo("Showing release: ${release}") - releases.add(release) - } else { - echo("Not showing release: ${release}") - } - } - timeout(time: 10, unit: 'MINUTES') { - release = input( - message: 'Please select the software version', - ok: 'Ok', - parameters: [ - choice(name: 'release', choices: releases, description: 'Which software release?') - ] - ) - echo "Chosen software version is: ${release}" - } - } - } - } - stage('Checkout release repo at selected software version') { - steps { - script { - SOURCEURL = RELEASE_REPO - SOURCEREVISION = "refs/tags/${release}" - SOURCEDIR = "${WORKSPACE}/whole" - configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - } - } - } - stage('Get release artifact info') { - steps { - script { - def mfs = autobuilderscript.getmetafolderscript() - releasemeta = mfs.metafromfolder(SOURCEDIR) - } - } - } - stage('Select which items from software release to publish') { - steps { - script { - publishitems = load("${autobuilder.commonscripts()}/publish/publishitems.groovy") - timeout(time: 10, unit: 'MINUTES') { - def itemsobject = publishitems.createItemSelector( - 'Select software items', - 'Software items', - releasemeta, - SOURCEDIR, - ITEMSDEFAULTLIST, - MAX_META_DEPTH, - '20', - 'PT_MULTI_SELECT') - selecteditems = input( - message: 'Select which items from the release are to be published', - ok: 'Ok', - parameters: [ - itemsobject.multiSelect() - ] - ) - echo "Chosen items are: ${selecteditems}" - } - } - } - } - stage('Publish components') { - steps { - script { - def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, SOURCEDIR, selecteditems, customer, release, MAX_META_DEPTH) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/publish.groovy b/Common_scripts/publish/publish.groovy deleted file mode 100644 index ad33df9..0000000 --- a/Common_scripts/publish/publish.groovy +++ /dev/null @@ -1,92 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) - } - } - } - stage('Clean folders') { - steps { - script { - cleanfolders = [ - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Checkout release repo') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - SOURCEURL = RELEASE_REPO - SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" - SOURCEDIR = "${WORKSPACE}/whole" - SOURCEROOTDIR = 'whole' - configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) - } - } - } - stage('Get release artifact meta') { - steps { - script { - def mfs = autobuilderscript.getmetafolderscript() - releasemeta = mfs.metafromfolder(SOURCEDIR) - } - } - } - stage('Publish components') { - steps { - script { - def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - def hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") - def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} - diff --git a/Common_scripts/publish/publishitems.groovy b/Common_scripts/publish/publishitems.groovy deleted file mode 100644 index 9f6c368..0000000 --- a/Common_scripts/publish/publishitems.groovy +++ /dev/null @@ -1,129 +0,0 @@ -import com.cwctravel.hudson.plugins.extended_choice_parameter.ExtendedChoiceParameterDefinition - -class PublishableItemsSelection { - def script - def ArrayList preselectedmenuitems - def ArrayList menuitemdescriptions - def ArrayList menuitemsfull - def ExtendedChoiceParameterDefinition multiselect - - PublishableItemsSelection(script) { - this.script = script - preselectedmenuitems = new ArrayList() - menuitemdescriptions = new ArrayList() - menuitemsfull = new ArrayList() - } - - def init( - String name, - String description, - releasemeta, - String releasemetapath, - String preselectedext, - String metalevelsstringext, - String visibleItemCountExt, - String controltype) { - outputSelectionfromPreselect(preselectedmenuitems, preselectedext) - def Integer metalevelsext = metalevelsstringext.toInteger() - def Integer visibleItemCount = visibleItemCountExt.toInteger() - def metaitems = releasemeta.releasewalkerunique(metalevelsext, '', releasemetapath) - for (key in metaitems.keySet()) { - def releaseinfo = metaitems[key] - outputDescription(menuitemdescriptions, releaseinfo) - outputSelection(menuitemsfull, releaseinfo) - } - - script.echo("name is: ${name}") - script.echo("description is: ${description}") - script.echo("menuitemsfull is: ${menuitemsfull}") - script.echo("preselectedmenuitems is: ${preselectedmenuitems}") - script.echo("menuitemdescriptions is: ${menuitemdescriptions}") - script.echo("visibleItemCount is: ${visibleItemCount}") - script.echo("controltype is: ${controltype}") - - def seperator = "," - visibleItemCount = Math.min(visibleItemCount, menuitemsfull.size()) - script.echo("visibleItemCount is now: ${visibleItemCount}") - - multiselect = new ExtendedChoiceParameterDefinition( - name, // String name - controltype, // String type - menuitemsfull.join(seperator), // String value - "project name", // String projectName - "", // String propertyFile - "", // String groovyScript - "", // String groovyScriptFile - "", // String bindings - "", // String groovyClasspath - "", // String propertyKey - preselectedmenuitems.join(seperator),// String defaultValue - "", // String defaultPropertyFile - "", // String defaultGroovyScript - "", // String defaultGroovyScriptFile - "", // String defaultBindings - "", // String defaultGroovyClasspath - "", // String defaultPropertyKey - menuitemdescriptions.join(seperator),// String descriptionPropertyValue - "", // String descriptionPropertyFile - "", // String descriptionGroovyScript - "", // String descriptionGroovyScriptFile - "", // String descriptionBindings - "", // String descriptionGroovyClasspath - "", // String descriptionPropertyKey - "", // String javascriptFile - "", // String javascript - false, // boolean saveJSONParameterToFile - false, // boolean quoteValue - visibleItemCount, // int visibleItemCount - description, // String description - seperator) // String multiSelectDelimiter - } - - def outputSelectionfromPreselect(menuitems, preselectedext) { - script.echo ("preselectext: ${preselectedext}") - def menuitemstemp = preselectedext.split(',') - script.echo ("Menuitemstemp: ${menuitemstemp}") - - for (menuitem in menuitemstemp) { - menuitems.add(menuitem.trim()) - } - } - - def outputDescription(menuitems, releaseinfo) { - def meta = releaseinfo.metainfo - - for (publishableitem in meta.publishableinfo) { - menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") - } - } - - def outputSelection(menuitems, releaseinfo) { - def meta = releaseinfo.metainfo - - for (publishableitem in meta.publishableinfo) { - menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") - } - } - - def multiSelect() { - return multiselect - } -} - -PublishableItemsSelection createItemSelector( - String name, - String description, - releasemeta, - String releasemetapath, - String preselectedext, - String metalevelsstringext, - String visibleItemCountExt = '20', - String type = 'PT_CHECKBOX') { // Could also be "PT_MULTI_SELECT" - PublishableItemsSelection pis = new PublishableItemsSelection(this) - - pis.init(name, description, releasemeta, releasemetapath, preselectedext, metalevelsstringext, visibleItemCountExt, type) - - return pis -} - -return this; diff --git a/Common_scripts/publishing/utils.groovy b/Common_scripts/publishing/utils.groovy deleted file mode 100644 index 2daf6f6..0000000 --- a/Common_scripts/publishing/utils.groovy +++ /dev/null @@ -1,27 +0,0 @@ - - -// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout -def allowed(String filepath) { - if (filepath.endsWith('/**') || filepath.endsWith('/')) { - def files = findFiles glob: filepath - if (files.length > 0) { - return - } else { - error "There are no files/folders under the publishable item: ${filepath}" - } - } - error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" -} - -// Generate a list of artifact files that are publishable to outside parties -def generatelist(meta, publishable) { - filelist = [] - for (filepath in publishable) { - if (filepath != meta.METAARTIFACTS) { - allowed(filepath) - } - meta.addPublishable(filepath) - } -} - -return this; diff --git a/Common_scripts/release/do.groovy b/Common_scripts/release/do.groovy deleted file mode 100644 index 05de91f..0000000 --- a/Common_scripts/release/do.groovy +++ /dev/null @@ -1,458 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -markunstable = false -unstablereason = "" - -summaryfolder = "${WORKSPACE}/summary" -summaryfile = "${summaryfolder}/summary.txt" -summarystring = "" -repoandtagsfile = "${summaryfolder}/repoandtags.txt" -repoandtagsstring = "" -releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" -tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() - -def resetfolders() { - dir(summaryfolder) { - deleteDir() - } - dir(releaserepofolder) { - deleteDir() - } - dir(tempfolder) { - deleteDir() - } -} - -def preconditions() { - markunstable = false - unstablereason = "" - summarystring = "" - writeFile(file: summaryfile, text: "Release summary: \n") - repoandtagsstring = "" - writeFile(file: repoandtagsfile, text: "Repository summary: \n") -} - -def setunstable(String reason) { - markunstable = true - unstablereason += reason + "\n" - echo "Marked build as unstable: Reason: ${reason}" -} - -def summaryappend(String summary) { - def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary) - summarystring += summary -} - -def repoandtagsappend(String summary) { - def String repoandtagscontents = readFile(file: repoandtagsfile) - writeFile(file: repoandtagsfile, text: repoandtagscontents + summary) - repoandtagsstring += summary -} - -def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = buildmeta.getNewMetaObject(this) - // See if there is meta data file from the current release - def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" - if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(checkoutfolder) - } - return previousreleasemeta -} - -def checkoutreleaserepo(buildmeta, String checkoutfolder) { - def String releaserepo = buildmeta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - repoandtagsappend(" Release repo: ${releaserepo}") - def previousreleasemeta = buildmeta.getNewMetaObject(this) - dir(releaserepofolder) { - sh("git clone ${releaserepo} ${checkoutfolder}") - } - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) - dir(checkoutfolder) { - // Clear all existing files - Note that this does not remove .git* - sh("git rm -r *") - } - return previousreleasemeta -} - -def checkoutsourcerepo(buildmeta, String checkoutfolder) { - def String sourcerepo = buildmeta.gitinfo.remoteurl - if (sourcerepo) { - repoandtagsappend(" Source repo: ${sourcerepo}") - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } - } else { - repoandtagsappend(" Source repo: None") - } -} - -def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { - // Copy relevant artifacts to temp folder - summaryappend("Artifacts: ") - def boolean result = true - for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { - def filterpath = artifactfilter - if (buildinfo.accumulatedpath) { - filterpath = buildinfo.accumulatedpath + '/' + artifactfilter - } - // Copy relevant artifacts to temp folder - echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - try { - copyArtifacts( - filter: filterpath, - projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - ], - target: tempfolder - ) - } catch (Exception e) { - echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" - result = false - summaryappend("Does not exist:") - } - if (result) { - // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath - dir(originalrootdir) { - sh("mv -v * ${repofolder}") - } - } - summaryappend("${artifactfilter},") - } - return result -} - -def releasegitadd() { - // Add anything new - sh("git add .") - summaryappend("Add,") -} - -def releasegitreset() { - // Add anything new - sh("git reset --hard") - sh("git clean -fdx") - summaryappend("Reset,") -} - -def releasegitcommit(String buildstamp) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") -} - -// Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { - String buildsha = buildmeta.gitinfo.gitrevision - String previoussha = previousreleasemeta.gitinfo.gitrevision - - def boolean shachanged = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" - - String buildstamp = buildmeta.buildstampinfo.buildstamp - String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp - - def boolean buildstampchanged = (buildstamp != previousbuildstamp) - echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" - - def boolean changed = (shachanged | buildstampchanged) - echo "changed: ${changed}" - - return changed -} - -def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { - def ArrayList taglist = new ArrayList() - taglist.add(buildstamp) - - def ArrayList othertaglist = otherreleasetags.split(',') - for (othertagitem in othertaglist) { - if (othertagitem?.trim()) { - def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}" - taglist.add(componentothertagitem) - } - } - return taglist -} - -def String releasegitgetshacode(String tag) { - def existingshacode = "" - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - existingshacode = readFile(file: gitrevparseoutputfile).trim() - } - return existingshacode -} - -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def String existingshacode = releasegitgetshacode(tag) - if (existingshacode) { // Tag exists - if (existingshacode != postcommitshacode) { // If the existing tag is not associated with the one expected - // This is a problem - The tag is already at another position in git repo - setunstable("The release tag ${tag} has been used at another commit other than one expected - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${postcommitshacode})") - summaryappend("(${tag} - Could not add!)") - } else { - summaryappend("(${tag} - Already added)") - } - } else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\"") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def boolean releasegittags(String postcommitshacode, ArrayList taglist) { - boolean tagadded = false - - summaryappend("Rel Tag ") - for (tag in taglist) { - if (releasegittag(postcommitshacode, tag)) { - echo "Tag added: ${tag} with sha: ${postcommitshacode}" - tagadded = true - } - } - return tagadded -} - -def boolean sourcegittag(String shacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != shacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it (Existing sha code: ${existingshacode} Post commit sha code: ${shacode})") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\" ${shacode}") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def sourcegittags(String buildsha, List taglist) { - boolean tagadded = false - - summaryappend(",Src Tag ") - for (tag in taglist) { - if (sourcegittag(buildsha, tag)) { - tagadded = true - } - } - return tagadded -} - -def releasegitpush(String dryrunpush ) { - sh("git remote -v") - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend(",Dry-Run-Push") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend(",Push") - } -} - -def sourcegitpush(String dryrunpush ) { - sh("git remote -v") - if (dryrunpush == "true") { - sh("git push -n --tags") - summaryappend(",Src-Dry-Run-Push") - setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags") - summaryappend(",Src Push") - } -} - -def boolean isalreadyreleased(buildmeta, tag) { - def boolean alreadyreleased = buildmeta.releaseinfo.released - if (alreadyreleased) { - echo "Component is already marked as released: ${alreadyreleased}" - } else { - String commitshacode = releasegitgetshacode(tag) - if (commitshacode) { - echo "Tag: ${tag} already exists in this repo therefore marking as already released" - alreadyreleased = true - } else { - alreadyreleased = false - } - } - return alreadyreleased -} - -def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { - summaryappend(" Git operations: ") - def String buildstamptag = buildmeta.releasetag() - - dir(releaserepo) { - def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Pre commit sha code: ${precommitshacode}" - - def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) - - def String postcommitshacode - def boolean changed = false - if (alreadyreleased) { - summaryappend("Already-released,") - postcommitshacode = releasegitgetshacode(buildstamptag) - echo "Previous release sha code: ${postcommitshacode}" - } else { - releasegitadd() - - changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamptag) - } else { - releasegitreset() - summaryappend("No-commit,") - } - - postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" - } - - ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) - def tagadded = releasegittags(postcommitshacode, taglist) - - if (changed || tagadded) { - releasegitpush(dryrunpush) - } - } - dir(sourcerepo) { - def String srcbuildstamptag = buildmeta.globalreleasetag() - def String buildsha = buildmeta.gitinfo.gitrevision - - if (buildsha) { - def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) - def tagadded = sourcegittags(buildsha, srctaglist) - if (tagadded) { - sourcegitpush(dryrunpush) - } - } else { - summaryappend(",No Src ") - } - } -} - -def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { - def buildmeta = buildinfo.metainfo - def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) - def String uniquefolder = "${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - - summaryappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") - repoandtagsappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") - def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) - - checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - - if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) - } - - summaryappend("\n") - repoandtagsappend("\n") -} - -def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { - def int md = maxdepth.toInteger() - resetfolders() - preconditions() - - // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) - echo "Unique components:" - def String versioncomponent = "" - for (key in componentReleaseInfo.keySet()) { - versioncomponent += "${key}\n" - } - echo versioncomponent - - // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) - def String versionuniquecomponent = '' - def String versionwithpathscomponent = '' - for (key in componentReleaseInfoVersionUnique.keySet()) { - versionuniquecomponent += "${key}\n" - versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" - } - echo 'Version unique components:' - echo versionuniquecomponent - - echo 'Version unique components with paths:' - echo versionwithpathscomponent - - if (DO_INTEGRITY_CHECK == "true") { - if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { - error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" - } - } - - for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { - error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" - } - } - - def toplevelid = buildmeta.componentidentifier() - // For each component - for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - echo "Releasing component: ${buildinfo.metainfo.componentidentifier()}" - if (buildinfo.metainfo.componentidentifier() != '___' && buildinfo.metainfo.componentidentifier() != '_') { - if (toplevelid != key) { - releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) - } - } - } - // Do the top level last - def toplevelbuildinfo = componentReleaseInfo[toplevelid] - releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) - - return [summarystring, markunstable, unstablereason] -} - -return this; diff --git a/Common_scripts/release/registerrepo.groovy b/Common_scripts/release/registerrepo.groovy deleted file mode 100644 index f565706..0000000 --- a/Common_scripts/release/registerrepo.groovy +++ /dev/null @@ -1,12 +0,0 @@ - -vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo) { - if (releaserepo?.trim()) { - vc.checkRepoExists(releaserepo) - } - meta.releaseinfo.set(releaserepo) -} - -return this; diff --git a/Common_scripts/release/release.groovy b/Common_scripts/release/release.groovy deleted file mode 100644 index 82af511..0000000 --- a/Common_scripts/release/release.groovy +++ /dev/null @@ -1,87 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - cleanfolders = [ - "${WORKSPACE}/build_artifacts", - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Get build artifact meta') { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - BUILD_ARTIFACTS = 'build' - - def importoptions = importer.getNewImportOptions() - importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) - importoptions.setincrementversionnumber(false) - importoptions.setfilter('meta/**') - importoptions.setappendtoparentmeta(false) - buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) - } - } - } - stage('Release components') { - steps { - script { - def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") - // Place the top level build stamp on all - OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" - (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) - if (markunstable) { - env.unstablereason = unstablereason - currentBuild.result = 'UNSTABLE' - } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def artifactlist = meta.generateartifacts() - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive("summary/**,${artifactlist}") - } - } - } - stage('Summarise') { - steps { - script { - echo "Summary:\n${summarystring}\n" - if (env.unstablereason) { - echo "Instability reason: ${env.unstablereason}" - } - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/release/tag.groovy b/Common_scripts/release/tag.groovy deleted file mode 100644 index 6155e7f..0000000 --- a/Common_scripts/release/tag.groovy +++ /dev/null @@ -1,39 +0,0 @@ - -def startswithspecialprefix(String releasetag) { - def RELEASETAG_PREFIXES = [ - 'superceded' : 'Misspelt - Indicates a later release has superseded the release with this prefix', - 'superseded' : 'Indicates a later release has superseded the release with this prefix', - 'erroneous' : 'The release was released unintentionally', - 'faulty' : 'The release was found to have a serious fault', - 'ignore' : 'The release should be ignored', - 'inmainline' : 'This release was associated with development branch which has now been merged into mainline', - ] - - def RELEASETAG_PREFIX_SEPERATOR = '_' - - def boolean hasspecialprefix = false - def String releasetagprefix = '' - def String prefixfound = '' - def String prefixmeaning = '' - def String tagwithoutprefix = releasetag - - for (def prefix in RELEASETAG_PREFIXES.keySet()) { - def meaning = RELEASETAG_PREFIXES.get(prefix) - if (releasetag.toLowerCase().startsWith(prefix)) { - prefixfound = prefix - hasspecialprefix = true - prefixmeaning = meaning - tagwithoutprefix = releasetag[(prefix.size() + 1)..-1] - break - } - } - - def results = [ 'hasspecialprefix' : hasspecialprefix, - 'prefixfound' : prefixfound, - 'tagwithoutprefix' : tagwithoutprefix, - 'meaning' : prefixmeaning, - ] - return results -} - -return this diff --git a/Common_scripts/remoteaccess/ftp.groovy b/Common_scripts/remoteaccess/ftp.groovy deleted file mode 100644 index 25b9143..0000000 --- a/Common_scripts/remoteaccess/ftp.groovy +++ /dev/null @@ -1,32 +0,0 @@ -import org.apache.commons.net.ftp.FTPClient - -class FTPSiteInfo { - String url - String username - String password - String workingdirectory - - FTPSiteInfo(String url, String username, String password, String workingdirectory) { - this.url = url - this.username = username - this.password = password - this.workingdirectory = workingdirectory - } -} - -FTPSiteInfo createFTPSiteInfo(String url, String username, String password, String workingdirectory) { - FTPSiteInfo ftpsiteinfo = new FTPSiteInfo(url, username, password, workingdirectory) - return ftpsiteinfo -} - -FTPClient createFTPClient(FTPSiteInfo ftpsiteinfo) { - FTPClient ftpclient = new FTPClient() - ftpclient.connect(ftpsiteinfo.url) - ftpclient.enterLocalPassiveMode() - ftpclient.login(ftpsiteinfo.username, ftpsiteinfo.password) - ftpclient.changeWorkingDirectory(ftpsiteinfo.workingdirectory) - - return ftpclient -} - -return this; diff --git a/Common_scripts/remoteaccess/hostedftp.groovy b/Common_scripts/remoteaccess/hostedftp.groovy deleted file mode 100644 index cf62d32..0000000 --- a/Common_scripts/remoteaccess/hostedftp.groovy +++ /dev/null @@ -1,28 +0,0 @@ -ftpremote = load("${autobuilder.commonscripts()}/remoteaccess/ftp.groovy") - -def getHostedFTPSiteInfo() { - hostedftpsiteinfo = - ftpremote.createFTPSiteInfo("ftp.hostedftp.com", - "ftp@globalinkjetsystems.com", - "ramshorn42", - "/Customers/") - return hostedftpsiteinfo -} - -def getCustomerList() { - def List customerfilenames = new ArrayList() - def List customers = new ArrayList() - - def ftpsiteinfo = getHostedFTPSiteInfo() - def ftpclient = ftpremote.createFTPClient(ftpsiteinfo) - - customerfilenames = ftpclient.listNames() - for(customer in customerfilenames) { - def String converted = new String(customer.getBytes("ISO-8859-1"), "UTF-8") - customers.add(converted) - } - ftpclient.disconnect() - return customers -} - -return this; \ No newline at end of file diff --git a/Common_scripts/somsoftware/sominfo.groovy b/Common_scripts/somsoftware/sominfo.groovy deleted file mode 100644 index a7fdc82..0000000 --- a/Common_scripts/somsoftware/sominfo.groovy +++ /dev/null @@ -1,31 +0,0 @@ -def somsoftwareinfostore = null - -/* Example expected JSON -{ - "Artifact info": { - "Location": "Jenkins job artifact", - "Job": "SOM_GIS_master/SOM_GIS_master_build", - "Job selector": "true", - "Release repo": "git@hydra:som_gis-releases/som_gis_whole.git", - "Release repo revision": "" - } -}*/ - -def String asJsonText() { - return writeJSON(returnText: true, json: somsoftwareinfostore, pretty: 1) -} - -def boolean exists() { - return (somsoftwareinfostore != null) -} - -def set(String jsontext) { - somsoftwareinfostore = readJSON(text: jsontext) - echo "Som software support info:\n${asJsonText()}" -} - -def get() { - return somsoftwareinfostore -} - -return this \ No newline at end of file diff --git a/Common_scripts/staticanalysis/ccm.groovy b/Common_scripts/staticanalysis/ccm.groovy deleted file mode 100644 index ae91cb0..0000000 --- a/Common_scripts/staticanalysis/ccm.groovy +++ /dev/null @@ -1,85 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - CCMURL = 'git@hydra:tools/ccm.git' - CCMBRANCH = 'origin/master' - CCMDIR = "${WORKSPACE}/ccm" - configurationhelper.gitCheckoutAndRegister(meta, 'ccm', CCMURL, CCMBRANCH, CCMDIR) - CCM = "${CCMDIR}/bin/CCM.exe" - } - } - } - stage('Checkout source') { - steps { - script { - SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) - } - } - } - stage("Clean exports") { - steps { - script { - RESULTS = 'results' - cleanfolders = [ - RESULTS, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Static analysis') { - steps { - script { - def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" - writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) - echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" - - dir(RESULTS) { - echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" - if (isUnix()) { - sh """#!/bin/bash - ${CCM}" "${ccmconfigurationfile}" > ccm.xml - """ - } else { - echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" - bat """ - ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml - """ - } - } - recordIssues(tools: [ - ccm(pattern: "**/ccm.xml", reportEncoding: 'UTF-8'), - ]) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/staticanalysis/cpd.groovy b/Common_scripts/staticanalysis/cpd.groovy deleted file mode 100644 index c04649c..0000000 --- a/Common_scripts/staticanalysis/cpd.groovy +++ /dev/null @@ -1,152 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHONBRANCH = 'origin/3.6.3_linux' - } else { - PYTHONBRANCH = 'origin/3.6.3_windows' - } - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - - JAVAURL = 'git@hydra:tools/openjdk.git' - if (isUnix()) { - JAVABRANCH = 'origin/openjdk-11.0.2_linux-x64_bin' - } else { - JAVABRANCH = 'origin/openjdk-11.0.2_windows-x64_bin' - } - JAVADIR = "${WORKSPACE}/java" - configurationhelper.gitCheckoutAndRegister(meta, 'java', JAVAURL, JAVABRANCH, JAVADIR) - JAVA = "${JAVADIR}/ver/bin" - if (isUnix()) { - env.PATH = JAVA + ':' + PATH - } else { - env.PATH = JAVA + ';' + PATH - } - - PPDURL = 'git@hydra:tools/pmd.git' - PPDBRANCH = 'origin/master' - PPDDIR = "${WORKSPACE}/ppd" - configurationhelper.gitCheckoutAndRegister(meta, 'ppd', PPDURL, PPDBRANCH, PPDDIR) - PPD = "${PPDDIR}/bin" - } - } - } - stage('Checkout source') { - steps { - script { - SOURCEREVISION = "${SOURCE_REVISION}" - SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - SRC = "${SOURCE_PATH}" - if (SOURCE_OFFSET_PATH) { - SRC += "/${SOURCE_OFFSET_PATH}" - } - } - } - } - stage("Clean exports") { - steps { - script { - STATICANALYSIS = 'staticanalysis' - cleanfolders = [ - STATICANALYSIS, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } - stage('Static analysis') { - steps { - script { - def PPDFOLDER = "${STATICANALYSIS}/ppd" - def EXCLUDE_PARAM = '' - if (EXCLUDE_PATHS) { - EXCLUDE_PARAM = '--exclude "' - def exclude_path_list = EXCLUDE_PATHS.split(',') - for (exclude_path in exclude_path_list) { - EXCLUDE_PARAM += "${SRC}/${exclude_path.trim()}," - } - EXCLUDE_PARAM += '"' - } - if (isUnix()) { - sh """#!/bin/bash - set -ev - - mkdir -p "${PPDFOLDER}" - export PMD_JAVA_OPTS=-Xmx65536m - "${PPD}/run.sh" cpd --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" - """ - } else { - bat """ - mkdir -p "${PPDFOLDER}" - set PMD_JAVA_OPTS=-Xmx2048m - "${PPD}/cpd.bat" --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml" - """ - } - recordIssues(tools: [ - cpd(pattern: "**/cpd.xml", reportEncoding: 'UTF-8'), - ]) - } - } - } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/staticanalysis/cppcheck.groovy b/Common_scripts/staticanalysis/cppcheck.groovy deleted file mode 100644 index d959770..0000000 --- a/Common_scripts/staticanalysis/cppcheck.groovy +++ /dev/null @@ -1,114 +0,0 @@ -pipeline { - agent { label AGENT_LABEL } - stages { - stage('Initial setup') { - steps { - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - PYTHONBRANCH = 'origin/3.6.3_windows' - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - } - } - stage('Checkout source') { - steps { - script { - SOURCEREVISION = "${SOURCE_REVISION}" - SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" - } - } - } - stage("Clean exports") { - steps { - script { - STATICANALYSIS = 'staticanalysis' - cleanfolders = [ - STATICANALYSIS, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } - stage('Static analysis') { - steps { - script { - def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" - dir(CPPCHECKFOLDER) { - echo "Executing: ${CPPCHECK} --xml --xml-version=2 ${SRC} 2> cppcheck.xml" - if (isUnix()) { - sh """#!/bin/bash - set -e - - "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml - """ - } else { - bat """ - "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml - """ - } - } - recordIssues(tools: [ - cppCheck(pattern: "**/cppcheck.xml", reportEncoding: 'UTF-8'), - ]) - } - } - } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - post { - always { - echo 'Post-build' - } - failure { - echo 'Failed build' - } - unstable { - echo 'Unstable build' - } - success { - echo 'Succeeded build' - } - } -} diff --git a/Common_scripts/test/testethernetproduct.groovy b/Common_scripts/test/testethernetproduct.groovy deleted file mode 100644 index 5e2ce53..0000000 --- a/Common_scripts/test/testethernetproduct.groovy +++ /dev/null @@ -1,204 +0,0 @@ -import jenkins.model.Jenkins - -pipeline { - agent { label AGENT_LABEL } - stages { - stage ("Initial Setup") { - steps{ - script { - meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage ("Checkout test environment") { - steps { - script { - toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") - - def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' - def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' - def TEST_ENVIRONMENT_REVISION = 'origin/master' - - def environmentoptions = toolsimporter.getNewToolOptions() - environmentoptions.toolName('Ethernet product test environment') - environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) - toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) - - TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" - def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") - TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] - echo "Selected environment:\n${TEST_ENVIRONMENT}" - } - } - } - stage ('Setup') { - agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } - stages { - stage ("Checkout dependencies") { - steps { - script { - def PYTHON_URL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHON_REVISION = 'origin/3.6.3_linux' - } - else { - PYTHON_REVISION = 'origin/3.8.10_windows_for_test' - } - PYTHONDIR = "${WORKSPACE}/python" - - def pythonimportoptions = toolsimporter.getNewToolOptions() - pythonimportoptions.toolName('Python') - pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) - toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) - if (isUnix()) { - env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" - } - else { - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - - def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' - TEST_SUITE_DIR = 'TestSuiteDir' - env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" - - def regressiontestimportoptions = toolsimporter.getNewToolOptions() - regressiontestimportoptions.toolName('Tests') - regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) - toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) - - env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" - - def resultimportoptions = toolsimporter.getNewToolOptions() - resultimportoptions.toolName('Test history') - resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) - toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) - - def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' - env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" - - def coreimportoptions = toolsimporter.getNewToolOptions() - coreimportoptions.toolName('Test core') - coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) - toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) - - def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' - def FTDI_SOURCE_REVISION = 'origin/Core-98' - FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" - - def ftdiimportoptions = toolsimporter.getNewToolOptions() - ftdiimportoptions.toolName('FTDI relay driver') - ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) - toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) - - def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' - HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" - - def resultprocessingoptions = toolsimporter.getNewToolOptions() - resultprocessingoptions.toolName('History processing') - resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) - toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) - } - } - } - stage ("Import application to test") { - steps { - script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - - APPLICATION_UNDER_TEST = 'application_under_test' - env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" - - def applicationimportoptions = importer.getNewImportOptions() - applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) - applicationimportoptions.setfilter('app_package/**,control_package/**') - applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) - - meta.productinfo.set(applicationmeta.productinfo) - meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" - meta.gitinfo.set(applicationmeta.gitinfo) - meta.buildstampinfo.set(applicationmeta.buildstampinfo) - } - } - } - stage("Clean") { - steps { - script { - LOGGING_ROOT = 'Logging' - env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" - cleanfolders = [ - GIS_TEST_OUTPUT_LOGGING_ROOT, - ] - cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) - } - } - } - stage ('Test') { - steps { - script { - lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { - if (POWER_CYCLE_HARDWARE == 'true') { - dir(FTDI_SOURCE_DIR) { - bat """ - "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True - """ - } - } - if (UPDATE_HARDWARE == 'true') { - dir(GIS_TEST_SUITE_CORE) { - bat """ - "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" - """ - rc = bat (script:""" - "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" - """, - returnStatus: true) - if (rc == 1) { - unstable('FAILED while inspecting state of hardware') - } - } - } - if (DO_TEST == 'true') { - rc = bat(script:""" - ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} - """, - returnStatus: true - ) - if (rc == 1) { - unstable('FAILED while running tests') - } - } - } - } - } - } - stage("Archive Test History") { - steps{ - script { - dir(HISTORY_FOLDER) { - rc = bat (script:""" - "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" - """, - returnStatus: true - ) - if (rc == 1) { - unstable('FAILED to process history') - } - } - } - } - } - stage("Archive test artifacts"){ - steps{ - script { - def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) - artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") - artifactarchiver.archive(meta, artifactlist) - } - } - } - } - } - } -} - diff --git a/Common_scripts/textutils/subst.groovy b/Common_scripts/textutils/subst.groovy deleted file mode 100644 index c0207a3..0000000 --- a/Common_scripts/textutils/subst.groovy +++ /dev/null @@ -1,8 +0,0 @@ - -def substitute(String filepath, pattern, String subst) { - fileText = readFile(file: filepath, encoding: 'UTF-8') - def converted = fileText.replaceAll(pattern, subst) - writeFile(file: filepath, text: converted, encoding: 'UTF-8') -} - -return this; \ No newline at end of file diff --git a/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig b/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig deleted file mode 100644 index ea618a3..0000000 --- a/Common_scripts/tool configurations/vs2017install/buildmachines.vsconfig +++ /dev/null @@ -1,25 +0,0 @@ -{ - "version": "1.0", - "components": [ - "Microsoft.VisualStudio.Workload.ManagedDesktop", - "Microsoft.VisualStudio.Workload.NativeDesktop", - "Microsoft.VisualStudio.Workload.NetWeb", - "Microsoft.VisualStudio.Workload.NetCoreTools", - "microsoft.net.componentgroup.targetingpacks.common", - "microsoft.componentgroup.blend", - "microsoft.visualstudio.component.entityframework", - "microsoft.visualstudio.component.debugger.justintime", - "microsoft.visualstudio.component.wcf.tooling", - "microsoft.visualstudio.component.vc.diagnostictools", - "microsoft.visualstudio.component.vc.cmake.project", - "microsoft.visualstudio.component.vc.testadapterforboosttest", - "microsoft.visualstudio.component.vc.testadapterforgoogletest", - "microsoft.visualstudio.component.vc.atlmfc", - "microsoft.visualstudio.component.appinsights.tools", - "microsoft.visualstudio.component.diagnostictools", - "microsoft.visualstudio.componentgroup.web.cloudtools", - "microsoft.visualstudio.component.aspnet45", - "microsoft.visualstudio.component.webdeploy", - "microsoft.visualstudio.component.nuget.buildtools" - ] -} \ No newline at end of file diff --git a/Common_scripts/versioncontrol/git.groovy b/Common_scripts/versioncontrol/git.groovy deleted file mode 100644 index 2b6fb09..0000000 --- a/Common_scripts/versioncontrol/git.groovy +++ /dev/null @@ -1,402 +0,0 @@ -import hudson.plugins.git.* -import hudson.plugins.git.extensions.* -import hudson.plugins.git.extensions.impl.* - -clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") - -def boolean isShallowClonedRepo(String gitrootpath) { - def boolean isshallowclone = true - - dir( gitrootpath) { - if (isUnix()) { - isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() - } - else { - isshallowclone = powershell(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() - } - echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" - } - return isshallowclone -} - -String getGitInfo(String workingSetRootPath) { - dir(workingSetRootPath) { - return clh.getCommandOutput('git rev-parse HEAD') - } -} - -ArrayList gettaglist(String workingSetRootPath) { - ArrayList taglist = new ArrayList() - dir(workingSetRootPath) { - def tagliststring = clh.getCommandOutput('git tag --sort=-version:refname') - taglist = tagliststring.split('\\n') - } - return taglist -} - -LinkedHashMap> getremoteshatagmap(String repositoryURL) { - def shatagliststring = clh.getCommandOutput("git ls-remote --tags --sort=-version:refname ${repositoryURL}") - def shataglist = shatagliststring.split('\\n') - - LinkedHashMap> shatagmap = new LinkedHashMap>() - for (shatag in shataglist) { - def shatagfields = shatag.split('\\t') - def sha = shatagfields[0].trim() - def tag = shatagfields[1].trim() - if (shatagmap.get(sha) == null) { - def ArrayList taglist = new ArrayList() - taglist.add(tag) - shatagmap.put(sha, taglist) - } else { - def taglist = shatagmap.get(sha) - taglist.add(tag) - shatagmap.put(sha, taglist) - } - } - shatagmap.each { sha, tags -> - echo "sha: ${sha} has tags: ${tags}" - } - return shatagmap -} - -@NonCPS -def ArrayList sparsecheckoutconfig(ArrayList paths) { - ArrayList sparsecheckoutpaths = new ArrayList() - for (path in paths) { - sparsecheckoutpaths.add(new SparseCheckoutPath(path)) - } - return sparsecheckoutpaths -} - -def String uniquereporootname(String gitURL) { - def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") - echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" - return uniquename -} - -// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo -def String checkouttoreporoot(String gitURL) { - def String reporootsubfolder - def String reporootenv = env.REPO_ROOT - if (!reporootenv) { - reporootsubfolder = "" - echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" - } else { - def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootname}" - - if (!fileExists(reporootsubfolder)) { - echo "Checking out ${gitURL} to ${reporootsubfolder}" - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: 'origin/master'] - ], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CheckoutOption', timeout: 180], - [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], - ], - submoduleCfg: [], - ]) - } else { - echo "Found ${gitURL} checked out at ${reporootsubfolder}" -/* - dir(reporootsubfolder) { - if (isUnix()) { - sh """ - git status - if [ \$? -ne 0 ]; then exit 1; fi - """ - } else { - powershell """ - git status - if (\$lastexitcode -ge 1) { exit 1 } - """ - } - } -*/ - } - } - return reporootsubfolder -} - -def checkgitlfsinstall() { - if (isUnix()) { - sh """ - git lfs install - if [ \$? -ne 0 ]; then exit 1; fi - """ - - } else { - powershell """ - git lfs install - if (\$lastexitcode -ge 1) { exit 1 } - """ - } -} - -//No credentials specified -//Cloning the remote Git repository -//Cloning repository git@hydra:print-server-releases/print_server_whole.git -//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 -//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git -//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git -// > git --version # timeout=10 -// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 -// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 -// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 -// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 -// Cleaning workspace -// > git rev-parse --verify HEAD # timeout=10 -// No valid HEAD. Skipping the resetting -// > git clean -fdx # timeout=10 -// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git -// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 -// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 -// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 -// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) -// Enabling Git LFS pull -// > git config core.sparsecheckout # timeout=10 -// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 -// > git config --get remote.origin.url # timeout=10 -// > git lfs pull origin # timeout=120 -// Commit message: "2.10180.7389.21125_7" -// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 - - -// > git rev-parse --is-inside-work-tree # timeout=10 -// Fetching changes from the remote Git repository -// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 -// Cleaning workspace -// > git rev-parse --verify HEAD # timeout=10 -// Resetting working tree -// > git reset --hard # timeout=10 -// > git clean -fdx # timeout=10 -// Using shallow fetch with depth 1 -// Fetching upstream changes from git@hydra:releases/print-server-installer.git -// > git --version # timeout=10 -// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 -// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 -// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 -// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) -// Enabling Git LFS pull -// > git config core.sparsecheckout # timeout=10 -// > git read-tree -mu HEAD # timeout=10 -// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 -// > git config --get remote.origin.url # timeout=10 -// > git lfs pull origin # timeout=120 -// Commit message: "2.10180.7389.21125_3" -// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 - -def String lfsinstallandreporoot(String gitURL) { - checkgitlfsinstall() - return checkouttoreporoot(gitURL) -} - -def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { - if (!sparseitemlist.isEmpty()) { - def itemstring = "Items in sparse item list:\n" - for (item in sparseitemlist) { - itemstring += item + "\n" - } - echo itemstring - } -} - -def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { - echo "Git: Checking out: URL: ${gitURL} Revision: ${revision} to: ${targetDir}" - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: revision] - ], - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], - [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], - [$class: 'PruneStaleBranch'], - ], - submoduleCfg: [], - ]) -} - -// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - def String reporootpath = lfsinstallandreporoot(gitURL) - showsparseitemlist(sparseitemlist) - docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) -} - -def gitinit(String targetDir, String reporootpath) { - if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist - echo "git init ${targetDir}" - if (isUnix()) { - sh """ - git init "${targetDir}" - """ - } else { - powershell """ - git init "${targetDir}" - """ - } - } - echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" - dir("${targetDir}/.git/objects/info") { - writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") - } -} - -def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { - def String sparseitemliststring = "" - if (!sparseitemlist.isEmpty()) { - for (item in sparseitemlist) { - sparseitemliststring += "${item}," - } - echo "git config lfs.fetchinclude ${sparseitemliststring}" - dir(targetDir) { - if (isUnix()) { - sh """ - git config lfs.fetchinclude "${sparseitemliststring}" - """ - } else { - powershell """ - git config lfs.fetchinclude "${sparseitemliststring}" - """ - } - } - } -} - -def gitsparserestore(String targetDir) { - echo "Clean out full fetch; Force a sparse fetch" - dir(targetDir) { - if (isUnix()) { - sh """ - git rm -rf . > /dev/null - git clean -fxd > /dev/null - git reset --hard > /dev/null - """ - } else { - powershell """ - git rm -rf . > \$null - git clean -fxd > \$null - git reset --hard > \$null - """ - } - } - } - -def List convertSparseItemsString(String itemstopublish) { - def List sparsepaths = [] - - boolean nonempty = itemstopublish?.trim() - if (nonempty) { - def String[] items = itemstopublish.trim().split(',') - def String allpaths = "" - for(String item : items) { - sparsepaths.add(item) - allpaths += "${item}\n" - } - echo "Selected item to publish:\n${allpaths}" - } - return sparsepaths -} - -def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - echo "Performing alternative checkout" - def String reporootpath = lfsinstallandreporoot(gitURL) - showsparseitemlist(sparseitemlist) - if (!sparseitemlist.isEmpty()) { - gitinit(targetDir, reporootpath) - setlfsfetchinclude(targetDir, sparseitemlist) - } - docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) - if (!sparseitemlist.isEmpty()) { - gitsparserestore(targetDir) - } -} - -def checkRepoExists(String gitURL) { - def int result = 0 - if (isUnix()) { - result = sh( script: """ - git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - """, - returnStatus: true - ) - } else { - result = powershell( script: """ - git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY - """, - returnStatus: true - ) - } - echo "git ls-remote return result: ${result}" - if (result > 0) { - error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") - } else { - echo "The release repo: ${gitURL} exists" - } -} - -String getSHAFromVersion(String versionFolder) { - dir (versionFolder) { - srcRevision = readFile("gitrevision.txt").trim() - } - return srcRevision -} - -boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { - def keys = versions.keySet() - if (keys.size() < 1) { - return false - } - if (keys.size() == 1) { - return true - } - else { - version = "" - for (key in keys) { - nextversion = versions.get(key) - if (version == "") { - version = nextversion - } - if (nextversion != version) { - return false - } - } - } - return true -} - -def stripgithistory(String rootfolder) { - echo "Stripping git history out of ${rootfolder}" - dir(rootfolder) { - if (isUnix()) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rf XX - find . -name ".gitignore" | xargs -iXX rm -rf XX - find . -name ".gitmodules" | xargs -iXX rm -rf XX - find . -iname ".gitattributes" | xargs -iXX rm -rf XX - find . -iname "*@tmp" | xargs -iXX rm -rf XX - """ - } else { - echo "Not supporting git stripping in Windows at the moment" - } - } -} - -return this diff --git a/Common_scripts/versionnumbering/do.groovy b/Common_scripts/versionnumbering/do.groovy deleted file mode 100644 index 208a60f..0000000 --- a/Common_scripts/versionnumbering/do.groovy +++ /dev/null @@ -1,23 +0,0 @@ - -def stamp(convertdetails) { - for (convertdetail in convertdetails) { - def filetoconvert = convertdetail[0] - def pattern = convertdetail[1] - def replacement = convertdetail[2] - echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" - def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - echo "Using python: ${PYTHON} and script: ${stampscript}" - if (isUnix()) { - sh """ - "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" - """ - } - else { - powershell """ - & "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" - """ - } - } -} - -return this; diff --git a/Common_scripts/versionnumbering/generate.groovy b/Common_scripts/versionnumbering/generate.groovy deleted file mode 100644 index 40413d7..0000000 --- a/Common_scripts/versionnumbering/generate.groovy +++ /dev/null @@ -1,187 +0,0 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - -def int maxversionnumber() { - def int MAXVERSIONNUMBER = 65535 - return MAXVERSIONNUMBER -} - -int incrementrevision(int current, int increment) { - def int newrevision = (current + increment) % (maxversionnumber() + 1) - echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" - return newrevision -} - -def getGitRevision(String gitrootpath, int basebuildnumber = 0) { - def int adjustedchangecount - def String gitrevision - def String remoteinfo - def String remoteurl - - def boolean isShallowCloned = gh.isShallowClonedRepo(gitrootpath) - if (isShallowCloned) { - error("Cannot generate a version number from a shallow cloned repo at ${gitrootpath}") - } - - dir( gitrootpath) { - def int changecount = 0 - if (isUnix()) { - changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() - } - else { - changecount = powershell(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() - } - echo "Change count: ${changecount}" - def int mvnp1 = maxversionnumber() + 1 - basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber - adjustedchangecount = incrementrevision(changecount, basebuildnumber) - echo "Adjusted change count: ${adjustedchangecount}" - if (isUnix()) { - gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() - } - else { - gitrevision = powershell(returnStdout: true, script: "git rev-parse HEAD").trim() - } - echo "Revision: ${gitrevision}" - if (isUnix()) { - remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() - } - else { - remoteinfo = powershell(returnStdout: true, script: "git remote show origin").trim() - } - echo "Remote info: ${remoteinfo}" - if (isUnix()) { - remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() - } - else { - remoteurl = powershell(returnStdout: true, script: "git config --get remote.origin.url").trim() - } - echo "Remote URL: ${remoteurl}" - } - return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] -} - -// Handle: -// origin/master -// origin/2.0.x -def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { - echo "Determining version number prefix from source branch name: ${sourcebranch}" - sourcebranch = sourcebranch.toLowerCase() - - def defaultversionprefix = "0.30000.0." - def splits = sourcebranch.split('/') - if (splits.size() < 1) { - echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" - return defaultversionprefix - } - - def mastertypeversionprefix = "0.20000.0." - sourcebranch = splits.last() - if (sourcebranch.toLowerCase().find("master") != null) { - echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" - return mastertypeversionprefix - } - - def splits2 = sourcebranch.split('\\.') - if (splits2.size() < 1) { - def thirdpartyversionprefix = "0.25000.0." - echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" - return thirdpartyversionprefix - } - - for (split in splits2) { - if (split == "x") { - echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" - return mastertypeversionprefix - } - } - - def integerstrings = sourcebranch.findAll(/\d+/) - def integers = [] - for (integerstring in integerstrings) { - if (integerstring.length() > 6) { - integerstring = integerstring.substring(0, 5) - } - int integer = integerstring.toInteger() - integer = integer % (maxversionnumber() + 1) - integers.add(integer) - } - - def versionprefix = "" - if (integers.size() < 1) { - echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" - versionprefix = defaultversionprefix - } - else { - def int integernumbercount = 0 - for (integer in integers) { - versionprefix += integer + "." - integernumbercount += 1 - if (integernumbercount > 2) { - break - } - } - if (integernumbercount == 0) { - echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" - versionprefix = defaultversionprefix - } - else if (integernumbercount == 1) { - versionprefix += "0.0." - } - else if (integernumbercount == 2) { - versionprefix += "0." - } - } - if (droptrailingdot == true) { - versionprefix = versionprefix.substring(0, versionprefix.length() - 1); - } - echo "Determined version prefix to be: ${versionprefix}" - return versionprefix -} - -String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { - def int gitrevisioncount - def String gitrevision - def String remoteinfo - def String remoteurl - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) - - def int scriptgitrevisioncount - def String scriptgitrevision - def String scriptremoteinfo - def String scriptremoteurl - (scriptgitrevisioncount, scriptgitrevision, scriptremoteinfo, scriptremoteurl) = getGitRevision(autobuilder.getautobuildpath()) - - def int totalrevisioncount = incrementrevision(gitrevisioncount, scriptgitrevisioncount) - - def versionprefix = getVersionPrefix(sourcerevision) - - def buildstamp = versionprefix + totalrevisioncount.toString() - echo "Build stamp is: ${buildstamp}" - - meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, totalrevisioncount, buildstamp) - - return buildstamp -} - -String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { - generate(meta, sourcerevision, sourcepath, basebuildnumber) - meta.buildstampinfo.override(pregeneratedbuildstamp) - - return pregeneratedbuildstamp -} - -String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { - def int bbn = basebuildnumber.trim().toInteger() - return generate(meta, sourcerevision, sourcepath, bbn) -} - -String copyfrom(meta, othermeta) { - // Deep copy - meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) - meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) - - return meta.buildstampinfo.buildstamp -} - -return this; \ No newline at end of file diff --git a/Common_scripts/versionstamping/.project b/Common_scripts/versionstamping/.project deleted file mode 100644 index 74928d6..0000000 --- a/Common_scripts/versionstamping/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - versionstamping - - - - - - org.python.pydev.PyDevBuilder - - - - - - org.python.pydev.pythonNature - - diff --git a/Common_scripts/versionstamping/.pydevproject b/Common_scripts/versionstamping/.pydevproject deleted file mode 100644 index 746d0f0..0000000 --- a/Common_scripts/versionstamping/.pydevproject +++ /dev/null @@ -1,8 +0,0 @@ - - - -/${PROJECT_DIR_NAME} - -python 3.0 -python35 - diff --git a/Common_scripts/versionstamping/versionstamper.py b/Common_scripts/versionstamping/versionstamper.py deleted file mode 100644 index 9243aa4..0000000 --- a/Common_scripts/versionstamping/versionstamper.py +++ /dev/null @@ -1,38 +0,0 @@ -import argparse -import sys -import shutil -from builtins import FileNotFoundError - -def main(argv): - parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") - - # Call once arguments to allow or disallow something - parser.add_argument('-f', '--filetoconvert', help='The file to convert') - parser.add_argument('-p', '--pattern', default='0.30000.0.0', help='The pattern to search for and replace') - parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') - - args = parser.parse_args(argv) - - filetoconvert = args.filetoconvert - pattern = args.pattern - substitute = args.substitute - - print("Converting instances of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) - - with open(filetoconvert,'r') as c: - fileread = c.read() - - count = 0 - with open(filetoconvert, 'w') as f: - print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) - converted = fileread.replace(pattern, substitute) - count = converted.count(substitute) - print("Confirmed %s instances of %s post-conversion"%(count,substitute)) - f.write(converted) - - if count == 0: - return 1 - return 0 - -if __name__ == "__main__": - sys.exit(main(sys.argv[1:]))