diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy new file mode 100644 index 0000000..d7b69d0 --- /dev/null +++ b/artifacts/archiver.groovy @@ -0,0 +1,15 @@ +enum Store { + JENKINS, + ARTIFACTSERVER, +} + +this.JENKINS = Store.JENKINS +this.ARTIFACTSERVER = Store.JENKINS + +def archive(artifactlist, Store store = JENKINS ) { + if (store == JENKINS) { + archiveArtifacts(artifactlist) + } +} + +return this \ No newline at end of file diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 67d13c3..4d93cc5 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,4 +1,3 @@ - // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, importer, subJobs) { def ArrayList artifactFolders = new ArrayList() diff --git a/autobuilder.groovy b/autobuilder.groovy index 71d9b6a..667f89a 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -34,8 +34,65 @@ def getmetafolderscript() { return metafolderscript } -// Start up boilderplate -def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { +def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { + def String altered = releaserepo.trim() + + if (releaserepovariantsuffix?.trim()) { + altered = altered.replace("-RELEASE_REPO_VARIANT_SUFFIX-", releaserepovariantsuffix) + } + return altered +} + +def setBuildType(meta, boolean dofullbuild = true) { + def doquickbuild = false + + echo "Build trigger cause: ${currentBuild.getBuildCauses()}" + if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { + echo "Build triggered by user" + if (!dofullbuild) { + echo "Marking as quick build as user wanted one" + doquickbuild = true + } + } else { + echo "Build triggered automatically" + if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { + echo "Marking as quick build as it was triggered by a commit" + doquickbuild = true + } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { + echo "Marking as full build as it was triggered by timer" + } + } + echo "Build type is ${doquickbuild ? "quick" : "full"}" + if (doquickbuild) { + unstable('This not a full build - a subset of the artifacts is not being generated') + } + meta.jenkinsbuildinfo.set(!doquickbuild) + return doquickbuild +} + +def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { + def meta = getmetafolderscript().metareset(script) + echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" + meta.productinfo.set(productname, subcomponentname, productvariant) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + for (p in params) { + meta.jenkinsbuildinfo.addParameter(p.key, p.value) + } + return meta +} + +// Start up boilerplate +def startbuild( + script, + String productname = PRODUCT_NAME, + String subcomponentname = SUBCOMPONENT_NAME, + String releaserepo = RELEASE_REPO, + String productvariant = '', + String autobuildpath = autobuildroot, + String commonscriptspath = commonscriptsroot + ) { echo "Starting...." autobuildroot = autobuildpath @@ -44,16 +101,13 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam conditionalclean() - def meta = getmetafolderscript().metareset(script) - echo "Product name: ${productname} Subcomponent name: ${subcomponentname}" - meta.productinfo.set(productname, subcomponentname) - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + def meta = setupmeta(script, productname, subcomponentname, productvariant) def rr = load("${commonscripts()}/release/registerrepo.groovy") - echo "Release repo: ${releaserepo}" - rr.registerrepo(meta, releaserepo) + def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) + + echo "Release repo: ${alteredreleaserepo}" + rr.registerrepo(meta, alteredreleaserepo) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) @@ -61,13 +115,12 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam return meta } -def start(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String jirakey = JIRA_KEY, String releaserepo = RELEASE_REPO, String repoRootSuffix = "", String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { - echo "DEPRECATED method: start(script, String productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, ...)" - return startbuild(script, productname, subcomponentname, releaserepo, autobuildpath, commonscriptspath) -} - -// Start up boilderplate -def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +// Start up boilerplate +def startnonbuild( + script, + autobuildpath = autobuildroot, + commonscriptspath = commonscriptsroot + ) { echo "Starting non-build job...." autobuildroot = autobuildpath @@ -76,11 +129,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() - def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("Non-build-product", "Non-build-subcomponent") - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/build/integrity.groovy b/build/integrity.groovy new file mode 100644 index 0000000..e93c00f --- /dev/null +++ b/build/integrity.groovy @@ -0,0 +1,113 @@ +import static groovy.json.JsonOutput.* + +def check(buildmeta) { + def results = [ + 'good': true, + 'comment': '', + ] + + // Walk the meta looking for different versions of the same components + def allmeta = buildmeta.allrelatedmeta() + echo "All meta:\n${allmeta.comment}" + + // Store build stamps associated with each product component; Store a counter against each product component version + def productcomponents = [:] + for (meta in allmeta.metacollection) { + def key = meta.componentidentifier() + + if (!productcomponents.containsKey(key)) { + productcomponents[key] = [:] + } + def buildstampnumber = meta.releasetag() + if (!productcomponents[key].containsKey(buildstampnumber)) { + productcomponents[key][buildstampnumber] = 1 + } else { + productcomponents[key][buildstampnumber] += 1 + } + } + + // Show what was collected + echo prettyPrint(toJson(productcomponents)) + + // Look for product components with more than one build stamp + for (pc in productcomponents.keySet()) { + def bsns = productcomponents[pc] + if (bsns.size() > 1) { + results.good = false + results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" + for (bsn in bsns.keySet()) { + results.comment += "${bsn}\n" + } + } + } + + // Store product components associated with each release git url + def releasegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.releaseinfo.releaserepoURL + + if (key) { + if (!releasegiturls.containsKey(key)) { + releasegiturls[key] = [:] + } + def componentidentifier = meta.componentidentifier() + if (!releasegiturls[key].containsKey(componentidentifier)) { + releasegiturls[key][componentidentifier] = 1 + } else { + releasegiturls[key][componentidentifier] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(releasegiturls)) + + // Look for release git urls being used by more than one product component + for (rgu in releasegiturls.keySet()) { + def cis = releasegiturls[rgu] + if (cis.size() > 1) { + results.good = false + results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" + for (ci in cis.keySet()) { + results.comment += "${ci}\n" + } + } + } + + // Store git commit shas associated with each source git url + def sourcegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.gitinfo.remoteurl + + if (key) { + if (!sourcegiturls.containsKey(key)) { + sourcegiturls[key] = [:] + } + def commitsha = meta.gitinfo.gitrevision + if (!sourcegiturls[key].containsKey(commitsha)) { + sourcegiturls[key][commitsha] = 1 + } else { + sourcegiturls[key][commitsha] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(sourcegiturls)) + + // Look for source git urls associated with more than one sha code + for (sgu in sourcegiturls.keySet()) { + def cs = sourcegiturls[sgu] + if (cs.size() > 1) { + results.good = false + results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" + for (c in cs.keySet()) { + results.comment += "${c}\n" + } + } + } + + return results +} + +return this \ No newline at end of file diff --git a/build/postactions.groovy b/build/postactions.groovy new file mode 100644 index 0000000..8b18908 --- /dev/null +++ b/build/postactions.groovy @@ -0,0 +1,225 @@ +/* Example post build info +{ + "Build outcomes": { + "Success": { + "Actions": { + "Deploy documentation": { + "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" + }, + "Static analysis": { + "Description": "Can be empty - Static analysis job to run when this build is successful", + "Job": "" + }, + "Unit test": { + "Description": "Can be empty - Unit test job to run when this build is successful", + "Job": "" + }, + "Regression test": { + "Description": "Can be empty - Regression test job to run when this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" + }, + "Update differences": { + "Description": "Can be empty - Update differences job to run when this build is successful", + "Job": "HMB_FD_HV_diff_releases" + }, + "Release to git": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "Job": "HMB_FD_HV_release_to_git" + } + } + }, + "Failed": { + "Actions": { + "Email": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ] + }, + "ExtEmail": { + "Description": "Send an email using Extended Email plugin", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ], + "Recipient providers": [ + "culprits", + "requestor", + "developers", + "upstreamDevelopers" + ] + } + } + } + } +} +*/ + +def getActionJobName(postbuildactions, String action, String alternativeparamname) { + def String jobname = '' + if (postbuildactions) { + try { + jobname = postbuildactions[action]['Job'] + } catch(Exception ex) { + echo "No post build actions found for action: ${action}" + jobname = '' + } + } else { + if (params[alternativeparamname] != null) { + if (params[alternativeparamname]) { + jobname = params[alternativeparamname] + } + } + } + if (jobname) { + def jobitem = Jenkins.instance.getItemByFullName(jobname) + if (jobitem) { + if (!jobitem.isBuildable()) { + jobname = '' + } + } else { + jobname = '' + } + } + // echo "Job name for action: ${action}: ${jobname}" + return jobname +} + +def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { + def jobname = getActionJobName(postbuildactions, action, alternativeparamname) + if (jobname) { + build( + wait: false, + job: jobname, + parameters: [ + string(name: jobnameparametername, value: JOB_NAME), + string(name: jobselectorparametername, value: buildselector), + ] + ) + } +} + +def readPostBuildInfo() { + def postbuildinfo = readJSON(text: POST_BUILD_INFO) + def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) + // echo "Post build info:\n${postbuildjsontext}" + + return postbuildinfo +} + +def performSuccessfulBuildActions() { + def postbuildactions + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Success/Actions" + postbuildactions = null + } + } + def String buildselector = "${BUILD_NUMBER}" + deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') + deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') +} + +def sendemails(emailinfo) { + def emailtolist = emailinfo['To'] + + if (emailtolist) { + def emailbody = """ + Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} + + Console output: ${BUILD_URL}console + """ + for(emailto in emailtolist) { + mail( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: emailbody, + to: emailto + ) + // echo "Send email to: ${emailto}" + } + } +} + +def getRecipientProviders(recipientProviderInfoList) { + def recipientProviderMap = [ + culprits: [$class: 'CulpritsRecipientProvider'], + requestor: [$class: 'RequesterRecipientProvider'], + developers: [$class: 'DevelopersRecipientProvider'], + upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], + ] + def recipientProviders = [] + + if (recipientProviderInfoList) { + for(rp in recipientProviderInfoList) { + if (recipientProviderMap.containsKey(rp)) { + recipientProviders.add(recipientProviderMap[rp]) + } + } + } + if (recipientProviders.size() == 0) { + recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] + } + // echo "recipientProviders: ${recipientProviders.toString()}" + return recipientProviders; +} + +def String getToList(toListInfo) { + def String toList = '' + + if (toListInfo) { + toList = toListInfo.join(',') + } + // echo "toList: ${toList}" + return toList +} + +def sendextemail(extmailinfo) { + def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) + def toList = getToList(extmailinfo['To']) + + emailext( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: '''${JELLY_SCRIPT,template="html"}''', + attachLog: true, + // attachmentsPattern: '', + compressLog: false, + // from : '', + mimeType: 'text/html', + // postsendScript: '', + // presendScript: '', + recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers + // replyTo: '', + saveOutput: false, + to: toList, + ) +} + +def performFailedBuildActions() { + def postbuildactions = null + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Failed/Actions" + postbuildactions = null + } + } + if (postbuildactions) { + if (postbuildactions['Email']) { + sendemails(postbuildactions['Email']) + } + if (postbuildactions['ExtEmail']) { + sendextemail(postbuildactions['ExtEmail']) + } + } +} + +return this; diff --git a/build/postactionstest.groovy b/build/postactionstest.groovy new file mode 100644 index 0000000..150f5d1 --- /dev/null +++ b/build/postactionstest.groovy @@ -0,0 +1,43 @@ +pipeline { + agent { label 'master' } + stages { + stage('Initial setup') { + steps { + script { + CLEAN_WORKSPACE = false + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Dummy build job') { + steps { + script { + echo "Execute dummy build job: ${CAUSE_ERROR}" + if (CAUSE_ERROR == 'true') { + error("Causing error") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() + } + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() + } + } + } +} diff --git a/check/build_meta.groovy b/check/build_meta.groovy index 34eed98..fb8a75d 100644 --- a/check/build_meta.groovy +++ b/check/build_meta.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -53,7 +53,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 9225100..1fd9078 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -227,7 +227,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { try { - timeout(time: 5, unit: 'MINUTES') { + timeout(time: 10, unit: 'MINUTES') { addToolchainPath(toolchainfolder) doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) } diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 2a7f1ab..7a89975 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,7 +1,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) + return meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) } def registerPipelineScriptConfiguration(meta) { @@ -12,7 +12,7 @@ def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, St def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) + return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this diff --git a/configuration/tools.groovy b/configuration/tools.groovy new file mode 100644 index 0000000..dbaeedb --- /dev/null +++ b/configuration/tools.groovy @@ -0,0 +1,59 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +class ToolOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + String m_toolName + + String m_gitURL + String m_revision + String m_repoRoot + int m_cloneDepth + boolean m_shallowClone + + ToolOptions(script) { + m_script = script + m_gitURL = '-' + m_revision = '-' + m_repoRoot = '' + m_cloneDepth = 1 + m_shallowClone = true + } + + void toolName(String name) { + m_toolName = name.trim() + } + + void toolRepo(String gitURL, String revision) { + m_gitURL = gitURL.trim() + m_revision = revision.trim() + } + + void repoRoot(String root) { + m_repoRoot = root.trim() + } + + void clonedepth(int depth) { + m_cloneDepth = depth + } + + void shallowclone(boolean shallow) { + m_shallowClone = shallow + } +} + +ToolOptions getNewToolOptions() { + return new ToolOptions(this) +} + +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) +} + +def getTool(meta, ToolOptions options, String targetDir) { + gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) + + registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/copy/all.groovy b/copy/all.groovy index b7c4607..9b95eb6 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -57,14 +57,16 @@ pipeline { def includes = '**/*' def excludes = "${IMPORTS}/*" echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" - // This does not work: archiveArtifacts artifacts: includes, excludes: excludes + // This does not work: artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + // This does not work: artifactarchiver.archive(artifacts: includes, excludes: excludes) // so cleanfolders = [ IMPORTS, 'autobuild', ] cleaner.clean(cleanfolders) - archiveArtifacts artifacts: includes + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifacts: includes) } } } diff --git a/copy/esw.groovy b/copy/esw.groovy index 2a63171..6ccb53e 100644 --- a/copy/esw.groovy +++ b/copy/esw.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -41,7 +41,8 @@ pipeline { rm -v -rf meta mv -v "${ESW}"/* . """ - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifactlist) } } } diff --git a/copy/filtered.groovy b/copy/filtered.groovy index feaa2ab..251de93 100644 --- a/copy/filtered.groovy +++ b/copy/filtered.groovy @@ -48,7 +48,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["filtered/**"]) - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifactlist) } } } diff --git a/copy/logic.groovy b/copy/logic.groovy index d1ece6d..7b20935 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -26,7 +26,10 @@ pipeline { stage('Copying logic artifacts') { steps { script { - importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC_RELEASE_REVISION, LOGICDEST) + sh('env') + def logicoptions = importer.getNewImportOptions() + logicoptions.source(LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_ARTIFACT_RELEASE_REPO, LOGIC_ARTIFACT_RELEASE_REVISION) + importer.artifactimport(meta, logicoptions, LOGICDEST) } } } @@ -40,7 +43,8 @@ pipeline { rm -v -rf diff mv -v "logic"/* . """ - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifactlist) } } } diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 4bcdcdc..60d9835 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -58,7 +58,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,reports/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,reports/**,${artifactlist}") } } } diff --git a/documentation/doxygen_deploy.groovy b/documentation/doxygen_deploy.groovy index 6642256..5054c02 100644 --- a/documentation/doxygen_deploy.groovy +++ b/documentation/doxygen_deploy.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/hardware/sominfo.groovy b/hardware/sominfo.groovy new file mode 100644 index 0000000..72a3ca9 --- /dev/null +++ b/hardware/sominfo.groovy @@ -0,0 +1,80 @@ +def sominfostore = null + +/* Example expected JSON +{ + "SomType": { + "Som_BA4": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", + "Release repo revision": "" + } + }, + "Som_FC5": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", + "Release repo revision": "" + } + } + } +} +*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: sominfostore, pretty: 1) +} + +def boolean exists() { + return (sominfostore != null) +} + +def set(String jsontext) { + sominfostore = readJSON(text: jsontext) + echo "Som support info:\n${asJsonText()}" +} + +def get() { + return sominfostore +} + +def somtypes() { + return sominfostore['SomType'] +} + +def artifactinfo(somtype) { + return somtype['Artifact info'] +} + +def artifactlocation(artifactinfo) { + return artifactinfo['Location'] +} + +def boolean isjoblocation(artifactinfo) { + return (artifactinfo['Location'] == 'Jenkins job artifact' || + artifactinfo['Location'] == 'Jenkins job artefact') +} + +def boolean isreleaserepolocation(artifactinfo) { + return artifactinfo['Location'] == 'Release repository' +} + +def artifactjob(artifactinfo) { + return [ + "name": artifactinfo['Job'], + "selector": "${artifactinfo['Job selector']}" + ] +} + +def releaserepo(artifactinfo) { + return [ + "repo": artifactinfo['Release repo'], + "revision": artifactinfo['Release repo revision'] + ] +} + +return this \ No newline at end of file diff --git a/import/importer.groovy b/import/importer.groovy index 8624421..0e2ca17 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -96,9 +96,21 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void jenkinsjob(String jobname, String buildnumberorselector) { + void jenkinsjob(String jobname, buildnumberorselector) { m_jobname = jobname.trim() - m_buildnumberorselector = buildnumberorselector.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: buildnumberorselector + ] + setsourceselect('Jenkins job artifact') + } + + void jenkinsjob(String jobname, int buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: "${buildnumberorselector}" + ] setsourceselect('Jenkins job artifact') } @@ -147,15 +159,14 @@ def artifactimportfrombuildjob(meta, importoptions, String destpath) { conditionalClean(importoptions, destpath) + echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" + echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" copyArtifacts( filter: formFilter(importoptions), excludes: importoptions.m_excludes, fingerprintArtifacts: importoptions.m_fingerprint, projectName: importoptions.m_jobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: importoptions.m_buildnumberorselector - ], + selector: importoptions.m_buildnumberorselector, flatten: importoptions.m_flatten, target: destpath ) @@ -276,7 +287,7 @@ def artifactimport(meta, boolean fingerprint = false, boolean nested = false) { echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) importoptions.setincrementversionnumber(incrementversionnumber) @@ -298,7 +309,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false) { echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(buildjob, " ${buildnumber}") importoptions.setincrementversionnumber(incrementversionnumber) @@ -319,7 +330,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false) { echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(buildjob, buildnumberorselector) importoptions.setincrementversionnumber(incrementversionnumber) @@ -330,4 +341,25 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } +// Import the artifacts of a named child component found below parent meta's hierarchy +def artifactimport(meta, parentmeta, String childproduct, String childsubcomponent, importoptions, String destinationfolder) { + // Import from parent meta + def childmeta = meta.getNewMetaObject() + def childmeta2 + echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" + def findresults = parentmeta.findAssociatedMeta(childproduct, childsubcomponent) + if (findresults.matchfound) { + childmeta = findresults.matchedmeta + if (childmeta.releaseinfo.released) { + importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) + } else { + importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) + } + childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) + } else { + error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") + } + return childmeta2 +} + return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index f600155..d67389f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -22,21 +22,23 @@ abstract class GISJsonSerialised implements Serializable { class ProductInfo extends GISJsonSerialised { String productname String subcomponentname + String variant - ProductInfo(script, productname = "", subcomponentname = "") { + ProductInfo(script, productname = "", subcomponentname = "", variant = "") { super(script) this.productname = productname this.subcomponentname = subcomponentname + this.variant = variant } - void set(String productname, String subcomponentname) { + void set(String productname, String subcomponentname, String variant = '') { // Make sure the fields can be used in git tags def int pstatus = 0 if (script.isUnix()) { pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - pstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${productname}\" > NUL 2> NUL", , returnStatus: true) + pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) } if (pstatus > 0) { script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") @@ -46,24 +48,45 @@ class ProductInfo extends GISJsonSerialised { sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - sstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${subcomponentname}\" > NUL 2> NUL", , returnStatus: true) + sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) } if (sstatus > 0) { script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") } + if (variant) { + def int vstatus = 0 + if (script.isUnix()) { + vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) + } + if (vstatus > 0) { + script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") + } + } this.productname = productname this.subcomponentname = subcomponentname + this.variant = variant + } + + void set(ProductInfo otherinfo) { + this.productname = otherinfo.productname + this.subcomponentname = otherinfo.subcomponentname + this.variant = otherinfo.variant } void deserialiseJson(rawmeta) { productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] + variant = rawmeta['variant'] ?: "" } LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, + 'variant' : variant, ] return data } @@ -85,6 +108,15 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + GitInfo(GitInfo another) { + super(another.script) + this.sourcepath = another.sourcepath + this.sourcerevision = another.sourcerevision + this.gitrevision = another.gitrevision + this.remoteurl = another.remoteurl + this.remoteinfo = another.remoteinfo + } + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { this.sourcepath = sourcepath this.sourcerevision = sourcerevision @@ -93,6 +125,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + void set(GitInfo otherinfo) { + this.sourcepath = otherinfo.sourcepath + this.sourcerevision = otherinfo.sourcerevision + this.gitrevision = otherinfo.gitrevision + this.remoteurl = otherinfo.remoteurl + this.remoteinfo = otherinfo.remoteinfo + } + void deserialiseJson(rawmeta) { sourcepath = rawmeta['sourcepath'] sourcerevision = rawmeta['sourcerevision'] @@ -113,25 +153,78 @@ class GitInfo extends GISJsonSerialised { } } +class JenkinsBuildParameter extends GISJsonSerialised { + private String name + private String valuetype + private String value + + JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { + super(script) + this.name = name + this.valuetype = valuetype + this.value = value + } + + void set(String name, String valuetype, String value) { + this.name = name + this.valuetype = valuetype + this.value = value + } + + void deserialiseJson(rawmeta) { + name = rawmeta['name'] + valuetype = rawmeta['valuetype'] + value = rawmeta['value'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'name' : name, + 'valuetype' : valuetype, + 'value' : value, + ] + return data + } +} + class JenkinsBuildInfo extends GISJsonSerialised { String buildjobname int buildnumber private String buildmachine private String builddate + ArrayList parameters + boolean isfullbuild - JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "", boolean isfullbuild = true) { super(script) this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine this.builddate = builddate + this.isfullbuild = isfullbuild + parameters = new ArrayList() } - void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine this.builddate = builddate + this.isfullbuild = isfullbuild + } + + void set(boolean isfullbuild) { + this.isfullbuild = isfullbuild + } + + boolean isFullBuild() { + return isfullbuild + } + + void addParameter(String name, value) { + def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) + parameters.add(p) + return p } void deserialiseJson(rawmeta) { @@ -139,6 +232,23 @@ class JenkinsBuildInfo extends GISJsonSerialised { buildnumber = rawmeta['buildnumber'] buildmachine = rawmeta['buildmachine'] builddate = rawmeta['builddate'] + isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] + parameters = new ArrayList() + if (rawmeta['isfullbuild'] != null) { + for (rawparameter in rawmeta['parameters']) { + def p = new JenkinsBuildParameter(script) + p.deserialiseJson(rawparameter) + parameters.add(p) + } + } + } + + List parametersserialised() { + def associated = [] + for (p in parameters) { + associated.add(p.serialisationdata()) + } + return associated } LinkedHashMap serialisationdata() { @@ -147,6 +257,8 @@ class JenkinsBuildInfo extends GISJsonSerialised { 'buildnumber' : buildnumber, 'buildmachine' : buildmachine, 'builddate' : builddate, + 'isfullbuild' : isfullbuild, + 'parameters' : parametersserialised(), ] return data } @@ -164,12 +276,37 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstamp = buildstamp } + BuildStampInfo(BuildStampInfo another) { + super(another.script) + this.buildstampprefix = another.buildstampprefix + this.revision = another.revision + this.buildstamp = another.buildstamp + } + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp } + void set(BuildStampInfo other) { + set(other.buildstampprefix, other.revision, other.buildstamp) + } + + // Override the buildstamp + void override(String buildstamp) { + this.buildstamp = buildstamp + script.echo("Build stamp overridden to: ${this.buildstamp}") + def s = this.buildstamp.split('\\.') + if (s.size() != 4) { + script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") + } + this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" + script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") + this.revision = s[3].toInteger() + script.echo("Revision overridden to: ${this.revision}") + } + String buildstampcommasep() { def String buildstampcommasep = buildstamp.replace(".", ",") return buildstampcommasep @@ -191,7 +328,7 @@ class BuildStampInfo extends GISJsonSerialised { changerevision(revision) } - void set(String buildstampprefix, String revision) { + void set(String buildstampprefix, String revision = '0') { set(buildstampprefix, revision.toInteger()) } @@ -508,6 +645,7 @@ class MetaInfo extends GISJsonSerialised { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") configurationinfo.add(ci) + return ci } void addCustomInfo(String customId, String customValue) { @@ -518,6 +656,10 @@ class MetaInfo extends GISJsonSerialised { return custominfo.getAt(customId.trim()) } + def isFullBuild() { + return jenkinsbuildinfo.isFullBuild() + } + List publishableinfoserialised() { def publishable = [] for (publishableitem in publishableinfo) { @@ -741,9 +883,46 @@ class MetaInfo extends GISJsonSerialised { return releasemetaversionuniquecollection } + def walkrelated(ArrayList metacollection, String comment = '', int level = 0) { + def results = [ + 'metacollection' : metacollection, + 'comment': comment + ] + results.metacollection.add(this) + results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + results = othermeta.walkrelated(results.metacollection, results.comment, level + 1) + } + return [ 'metacollection' : results.metacollection, 'comment': results.comment ] + } + + def allrelatedmeta() { + ArrayList metacollection = new ArrayList() + return walkrelated(metacollection) + } + MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } + + def findAssociatedMeta(String productname, String subcomponentname) { + def results = [ + 'matchedmeta' : getNewMetaObject(this), + 'matchfound' : false + ] + def walkresults = allrelatedmeta() + for (dependentmeta in walkresults.metacollection) { + if (dependentmeta.productinfo.productname == productname && + dependentmeta.productinfo.subcomponentname == subcomponentname) { + script.echo("Found matching component: ${dependentmeta.globalreleasetag()}") + results.matchedmeta = dependentmeta + results.matchfound = true + break + } + } + return results + } } MetaInfo metareset(script) { diff --git a/publish/package.groovy b/publish/package.groovy index 3bd9565..958a614 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 21de708..58c72eb 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -147,7 +147,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/publish/publish.groovy b/publish/publish.groovy index 7beb134..ad33df9 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -58,7 +58,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/release/do.groovy b/release/do.groovy index 649e279..e0b9471 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -235,6 +235,7 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) summaryappend("Rel Tag ") for (tag in taglist) { if (releasegittag(postcommitshacode, tag)) { + echo "Tag added: ${tag} with sha: ${postcommitshacode}" tagadded = true } } diff --git a/release/release.groovy b/release/release.groovy index e51a174..42357fc 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -1,11 +1,11 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { script { autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") - meta = autobuilderscript.startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO) + meta = autobuilderscript.startnonbuild(this) } } } @@ -56,7 +56,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index f0e5b80..ae91cb0 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -13,12 +13,6 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - PYTHONURL = 'git@hydra:tools/python.git' - PYTHONBRANCH = 'origin/3.6.3_windows' - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - CCMURL = 'git@hydra:tools/ccm.git' CCMBRANCH = 'origin/master' CCMDIR = "${WORKSPACE}/ccm" @@ -30,49 +24,40 @@ pipeline { stage('Checkout source') { steps { script { - SOURCEREVISION = "${SOURCE_REVISION}" SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) } } } stage("Clean exports") { steps { script { - STATICANALYSIS = 'staticanalysis' + RESULTS = 'results' cleanfolders = [ - STATICANALYSIS, + RESULTS, ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } stage('Static analysis') { steps { script { - def CCMOLDER = "${STATICANALYSIS}/ccm" - dir(CCMOLDER) { - echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" + def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" + writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) + echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" + + dir(RESULTS) { + echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" if (isUnix()) { sh """#!/bin/bash - find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \\; - # Uses mono - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + ${CCM}" "${ccmconfigurationfile}" > ccm.xml """ } else { + echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" bat """ - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml """ } } @@ -82,25 +67,6 @@ pipeline { } } } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" - } - } - } } post { always { diff --git a/staticanalysis/cpd.groovy b/staticanalysis/cpd.groovy index 6f3a33f..f9ddfcb 100644 --- a/staticanalysis/cpd.groovy +++ b/staticanalysis/cpd.groovy @@ -129,7 +129,8 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifactlist) } } } diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index bdc6a2a..696f11d 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -91,7 +91,8 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifactlist) } } } diff --git a/test/testethernetproduct.groovy b/test/testethernetproduct.groovy new file mode 100644 index 0000000..f45cf19 --- /dev/null +++ b/test/testethernetproduct.groovy @@ -0,0 +1,204 @@ +import jenkins.model.Jenkins + +pipeline { + agent { label AGENT_LABEL } + stages { + stage ("Initial Setup") { + steps{ + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage ("Checkout test environment") { + steps { + script { + toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") + + def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' + def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' + def TEST_ENVIRONMENT_REVISION = 'origin/master' + + def environmentoptions = toolsimporter.getNewToolOptions() + environmentoptions.toolName('Ethernet product test environment') + environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) + + TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" + def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") + TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] + echo "Selected environment:\n${TEST_ENVIRONMENT}" + } + } + } + stage ('Setup') { + agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } + stages { + stage ("Checkout dependencies") { + steps { + script { + def PYTHON_URL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHON_REVISION = 'origin/3.6.3_linux' + } + else { + PYTHON_REVISION = 'origin/3.8.10_windows_for_test' + } + PYTHONDIR = "${WORKSPACE}/python" + + def pythonimportoptions = toolsimporter.getNewToolOptions() + pythonimportoptions.toolName('Python') + pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) + toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + + def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' + TEST_SUITE_DIR = 'TestSuiteDir' + env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" + + def regressiontestimportoptions = toolsimporter.getNewToolOptions() + regressiontestimportoptions.toolName('Tests') + regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) + + env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" + + def resultimportoptions = toolsimporter.getNewToolOptions() + resultimportoptions.toolName('Test history') + resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) + toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) + + def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' + env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" + + def coreimportoptions = toolsimporter.getNewToolOptions() + coreimportoptions.toolName('Test core') + coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) + toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) + + def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' + def FTDI_SOURCE_REVISION = 'origin/Core-98' + FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" + + def ftdiimportoptions = toolsimporter.getNewToolOptions() + ftdiimportoptions.toolName('FTDI relay driver') + ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) + toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) + + def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' + HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" + + def resultprocessingoptions = toolsimporter.getNewToolOptions() + resultprocessingoptions.toolName('History processing') + resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) + toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) + } + } + } + stage ("Import application to test") { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + APPLICATION_UNDER_TEST = 'application_under_test' + env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" + + def applicationimportoptions = importer.getNewImportOptions() + applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) + applicationimportoptions.setfilter('app_package/**,control_package/**') + applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) + + meta.productinfo.set(applicationmeta.productinfo) + meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" + meta.gitinfo.set(applicationmeta.gitinfo) + meta.buildstampinfo.set(applicationmeta.buildstampinfo) + } + } + } + stage("Clean") { + steps { + script { + LOGGING_ROOT = 'Logging' + env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" + cleanfolders = [ + GIS_TEST_OUTPUT_LOGGING_ROOT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage ('Test') { + steps { + script { + lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { + if (POWER_CYCLE_HARDWARE == 'true') { + dir(FTDI_SOURCE_DIR) { + bat """ + "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True + """ + } + } + if (UPDATE_HARDWARE == 'true') { + dir(GIS_TEST_SUITE_CORE) { + bat """ + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" + """ + rc = bat (script:""" + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" + """, + returnStatus: true) + if (rc == 1) { + unstable('FAILED while inspecting state of hardware') + } + } + } + if (DO_TEST == 'true') { + rc = bat(script:""" + ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED while running tests') + } + } + } + } + } + } + stage("Archive Test History") { + steps{ + script { + dir(HISTORY_FOLDER) { + rc = bat (script:""" + "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED to process history') + } + } + } + } + } + stage("Archive test artifacts"){ + steps{ + script { + def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(artifactlist) + } + } + } + } + } + } +} + diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 50f4025..288d5db 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -194,7 +194,6 @@ def docheckout(String gitURL, String revision, int cloneDepth, String reporootpa branches: [ [name: revision] ], - doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'GitLFSPull',timeout: 25], [$class: 'CleanBeforeCheckout',timeout: 20], @@ -202,6 +201,7 @@ def docheckout(String gitURL, String revision, int cloneDepth, String reporootpa [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'PruneStaleBranch'], ], submoduleCfg: [], ]) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 405b7be..5e974e2 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -98,6 +98,9 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { def integerstrings = sourcebranch.findAll(/\d+/) def integers = [] for (integerstring in integerstrings) { + if (integerstring.length() > 6) { + integerstring = integerstring.substring(0, 5) + } int integer = integerstring.toInteger() integer = integer % (maxversionnumber() + 1) integers.add(integer) @@ -154,14 +157,22 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { + generate(meta, sourcerevision, sourcepath, basebuildnumber) + meta.buildstampinfo.override(pregeneratedbuildstamp) + + return pregeneratedbuildstamp +} + String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { def int bbn = basebuildnumber.trim().toInteger() return generate(meta, sourcerevision, sourcepath, bbn) } String copyfrom(meta, othermeta) { - meta.gitinfo = othermeta.gitinfo - meta.buildstampinfo = othermeta.buildstampinfo + // Deep copy + meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) + meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) return meta.buildstampinfo.buildstamp }