diff --git a/autobuild/Common_scripts/artifacts/archiver.groovy b/autobuild/Common_scripts/artifacts/archiver.groovy new file mode 100644 index 0000000..bafba02 --- /dev/null +++ b/autobuild/Common_scripts/artifacts/archiver.groovy @@ -0,0 +1,17 @@ +enum Store { + JENKINS, + ARTIFACTSERVER, +} + +this.JENKINS = Store.JENKINS +this.ARTIFACTSERVER = Store.JENKINS + +def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { + echo "Archiving artifacts to store: ${store}" + meta.copyrelatedmeta() + if (store == JENKINS) { + archiveArtifacts(artifacts: artifactlist) + } +} + +return this \ No newline at end of file diff --git a/autobuild/Common_scripts/artifacts/metaexport.groovy b/autobuild/Common_scripts/artifacts/metaexport.groovy index 67d13c3..a40e9a8 100644 --- a/autobuild/Common_scripts/artifacts/metaexport.groovy +++ b/autobuild/Common_scripts/artifacts/metaexport.groovy @@ -1,18 +1,22 @@ - // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, importer, subJobs) { def ArrayList artifactFolders = new ArrayList() for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { - def subJobName = subJobObject.getProjectName() + def subJobName = subJobObject.getFullProjectName() def subJobNumber = subJobObject.getNumber() def destFolder = subJob[1] def boolean incrementversionnumber = subJob[2] dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(subJobName, subJobNumber) + importoptions.setincrementversionnumber(incrementversionnumber) + importer.artifactimport(meta, importoptions, destFolder) + artifactFolders.add("${destFolder}/**") } } diff --git a/autobuild/Common_scripts/autobuilder.groovy b/autobuild/Common_scripts/autobuilder.groovy index 2879f11..667f89a 100644 --- a/autobuild/Common_scripts/autobuilder.groovy +++ b/autobuild/Common_scripts/autobuilder.groovy @@ -43,8 +43,56 @@ def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { return altered } -// Start up boilderplate -def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String productvariant = '', String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { +def setBuildType(meta, boolean dofullbuild = true) { + def doquickbuild = false + + echo "Build trigger cause: ${currentBuild.getBuildCauses()}" + if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { + echo "Build triggered by user" + if (!dofullbuild) { + echo "Marking as quick build as user wanted one" + doquickbuild = true + } + } else { + echo "Build triggered automatically" + if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { + echo "Marking as quick build as it was triggered by a commit" + doquickbuild = true + } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { + echo "Marking as full build as it was triggered by timer" + } + } + echo "Build type is ${doquickbuild ? "quick" : "full"}" + if (doquickbuild) { + unstable('This not a full build - a subset of the artifacts is not being generated') + } + meta.jenkinsbuildinfo.set(!doquickbuild) + return doquickbuild +} + +def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { + def meta = getmetafolderscript().metareset(script) + echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" + meta.productinfo.set(productname, subcomponentname, productvariant) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + for (p in params) { + meta.jenkinsbuildinfo.addParameter(p.key, p.value) + } + return meta +} + +// Start up boilerplate +def startbuild( + script, + String productname = PRODUCT_NAME, + String subcomponentname = SUBCOMPONENT_NAME, + String releaserepo = RELEASE_REPO, + String productvariant = '', + String autobuildpath = autobuildroot, + String commonscriptspath = commonscriptsroot + ) { echo "Starting...." autobuildroot = autobuildpath @@ -53,12 +101,7 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam conditionalclean() - def meta = getmetafolderscript().metareset(script) - echo "Product name: ${productname} Subcomponent name: ${subcomponentname}" - meta.productinfo.set(productname, subcomponentname, productvariant) - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + def meta = setupmeta(script, productname, subcomponentname, productvariant) def rr = load("${commonscripts()}/release/registerrepo.groovy") def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) @@ -72,13 +115,12 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam return meta } -def start(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String jirakey = JIRA_KEY, String releaserepo = RELEASE_REPO, String repoRootSuffix = "", String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { - echo "DEPRECATED method: start(script, String productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, ...)" - return startbuild(script, productname, subcomponentname, releaserepo, autobuildpath, commonscriptspath) -} - -// Start up boilderplate -def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +// Start up boilerplate +def startnonbuild( + script, + autobuildpath = autobuildroot, + commonscriptspath = commonscriptsroot + ) { echo "Starting non-build job...." autobuildroot = autobuildpath @@ -87,11 +129,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() - def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("Non-build-product", "Non-build-subcomponent") - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/autobuild/Common_scripts/build/integrity.groovy b/autobuild/Common_scripts/build/integrity.groovy new file mode 100644 index 0000000..e93c00f --- /dev/null +++ b/autobuild/Common_scripts/build/integrity.groovy @@ -0,0 +1,113 @@ +import static groovy.json.JsonOutput.* + +def check(buildmeta) { + def results = [ + 'good': true, + 'comment': '', + ] + + // Walk the meta looking for different versions of the same components + def allmeta = buildmeta.allrelatedmeta() + echo "All meta:\n${allmeta.comment}" + + // Store build stamps associated with each product component; Store a counter against each product component version + def productcomponents = [:] + for (meta in allmeta.metacollection) { + def key = meta.componentidentifier() + + if (!productcomponents.containsKey(key)) { + productcomponents[key] = [:] + } + def buildstampnumber = meta.releasetag() + if (!productcomponents[key].containsKey(buildstampnumber)) { + productcomponents[key][buildstampnumber] = 1 + } else { + productcomponents[key][buildstampnumber] += 1 + } + } + + // Show what was collected + echo prettyPrint(toJson(productcomponents)) + + // Look for product components with more than one build stamp + for (pc in productcomponents.keySet()) { + def bsns = productcomponents[pc] + if (bsns.size() > 1) { + results.good = false + results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" + for (bsn in bsns.keySet()) { + results.comment += "${bsn}\n" + } + } + } + + // Store product components associated with each release git url + def releasegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.releaseinfo.releaserepoURL + + if (key) { + if (!releasegiturls.containsKey(key)) { + releasegiturls[key] = [:] + } + def componentidentifier = meta.componentidentifier() + if (!releasegiturls[key].containsKey(componentidentifier)) { + releasegiturls[key][componentidentifier] = 1 + } else { + releasegiturls[key][componentidentifier] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(releasegiturls)) + + // Look for release git urls being used by more than one product component + for (rgu in releasegiturls.keySet()) { + def cis = releasegiturls[rgu] + if (cis.size() > 1) { + results.good = false + results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" + for (ci in cis.keySet()) { + results.comment += "${ci}\n" + } + } + } + + // Store git commit shas associated with each source git url + def sourcegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.gitinfo.remoteurl + + if (key) { + if (!sourcegiturls.containsKey(key)) { + sourcegiturls[key] = [:] + } + def commitsha = meta.gitinfo.gitrevision + if (!sourcegiturls[key].containsKey(commitsha)) { + sourcegiturls[key][commitsha] = 1 + } else { + sourcegiturls[key][commitsha] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(sourcegiturls)) + + // Look for source git urls associated with more than one sha code + for (sgu in sourcegiturls.keySet()) { + def cs = sourcegiturls[sgu] + if (cs.size() > 1) { + results.good = false + results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" + for (c in cs.keySet()) { + results.comment += "${c}\n" + } + } + } + + return results +} + +return this \ No newline at end of file diff --git a/autobuild/Common_scripts/build/postactions.groovy b/autobuild/Common_scripts/build/postactions.groovy new file mode 100644 index 0000000..8b18908 --- /dev/null +++ b/autobuild/Common_scripts/build/postactions.groovy @@ -0,0 +1,225 @@ +/* Example post build info +{ + "Build outcomes": { + "Success": { + "Actions": { + "Deploy documentation": { + "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" + }, + "Static analysis": { + "Description": "Can be empty - Static analysis job to run when this build is successful", + "Job": "" + }, + "Unit test": { + "Description": "Can be empty - Unit test job to run when this build is successful", + "Job": "" + }, + "Regression test": { + "Description": "Can be empty - Regression test job to run when this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" + }, + "Update differences": { + "Description": "Can be empty - Update differences job to run when this build is successful", + "Job": "HMB_FD_HV_diff_releases" + }, + "Release to git": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "Job": "HMB_FD_HV_release_to_git" + } + } + }, + "Failed": { + "Actions": { + "Email": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ] + }, + "ExtEmail": { + "Description": "Send an email using Extended Email plugin", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ], + "Recipient providers": [ + "culprits", + "requestor", + "developers", + "upstreamDevelopers" + ] + } + } + } + } +} +*/ + +def getActionJobName(postbuildactions, String action, String alternativeparamname) { + def String jobname = '' + if (postbuildactions) { + try { + jobname = postbuildactions[action]['Job'] + } catch(Exception ex) { + echo "No post build actions found for action: ${action}" + jobname = '' + } + } else { + if (params[alternativeparamname] != null) { + if (params[alternativeparamname]) { + jobname = params[alternativeparamname] + } + } + } + if (jobname) { + def jobitem = Jenkins.instance.getItemByFullName(jobname) + if (jobitem) { + if (!jobitem.isBuildable()) { + jobname = '' + } + } else { + jobname = '' + } + } + // echo "Job name for action: ${action}: ${jobname}" + return jobname +} + +def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { + def jobname = getActionJobName(postbuildactions, action, alternativeparamname) + if (jobname) { + build( + wait: false, + job: jobname, + parameters: [ + string(name: jobnameparametername, value: JOB_NAME), + string(name: jobselectorparametername, value: buildselector), + ] + ) + } +} + +def readPostBuildInfo() { + def postbuildinfo = readJSON(text: POST_BUILD_INFO) + def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) + // echo "Post build info:\n${postbuildjsontext}" + + return postbuildinfo +} + +def performSuccessfulBuildActions() { + def postbuildactions + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Success/Actions" + postbuildactions = null + } + } + def String buildselector = "${BUILD_NUMBER}" + deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') + deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') +} + +def sendemails(emailinfo) { + def emailtolist = emailinfo['To'] + + if (emailtolist) { + def emailbody = """ + Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} + + Console output: ${BUILD_URL}console + """ + for(emailto in emailtolist) { + mail( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: emailbody, + to: emailto + ) + // echo "Send email to: ${emailto}" + } + } +} + +def getRecipientProviders(recipientProviderInfoList) { + def recipientProviderMap = [ + culprits: [$class: 'CulpritsRecipientProvider'], + requestor: [$class: 'RequesterRecipientProvider'], + developers: [$class: 'DevelopersRecipientProvider'], + upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], + ] + def recipientProviders = [] + + if (recipientProviderInfoList) { + for(rp in recipientProviderInfoList) { + if (recipientProviderMap.containsKey(rp)) { + recipientProviders.add(recipientProviderMap[rp]) + } + } + } + if (recipientProviders.size() == 0) { + recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] + } + // echo "recipientProviders: ${recipientProviders.toString()}" + return recipientProviders; +} + +def String getToList(toListInfo) { + def String toList = '' + + if (toListInfo) { + toList = toListInfo.join(',') + } + // echo "toList: ${toList}" + return toList +} + +def sendextemail(extmailinfo) { + def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) + def toList = getToList(extmailinfo['To']) + + emailext( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: '''${JELLY_SCRIPT,template="html"}''', + attachLog: true, + // attachmentsPattern: '', + compressLog: false, + // from : '', + mimeType: 'text/html', + // postsendScript: '', + // presendScript: '', + recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers + // replyTo: '', + saveOutput: false, + to: toList, + ) +} + +def performFailedBuildActions() { + def postbuildactions = null + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Failed/Actions" + postbuildactions = null + } + } + if (postbuildactions) { + if (postbuildactions['Email']) { + sendemails(postbuildactions['Email']) + } + if (postbuildactions['ExtEmail']) { + sendextemail(postbuildactions['ExtEmail']) + } + } +} + +return this; diff --git a/autobuild/Common_scripts/build/postactionstest.groovy b/autobuild/Common_scripts/build/postactionstest.groovy new file mode 100644 index 0000000..150f5d1 --- /dev/null +++ b/autobuild/Common_scripts/build/postactionstest.groovy @@ -0,0 +1,43 @@ +pipeline { + agent { label 'master' } + stages { + stage('Initial setup') { + steps { + script { + CLEAN_WORKSPACE = false + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Dummy build job') { + steps { + script { + echo "Execute dummy build job: ${CAUSE_ERROR}" + if (CAUSE_ERROR == 'true') { + error("Causing error") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() + } + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() + } + } + } +} diff --git a/autobuild/Common_scripts/check/build_meta.groovy b/autobuild/Common_scripts/check/build_meta.groovy index 34eed98..fb8a75d 100644 --- a/autobuild/Common_scripts/check/build_meta.groovy +++ b/autobuild/Common_scripts/check/build_meta.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -53,7 +53,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/autobuild/Common_scripts/check/do_meta.groovy b/autobuild/Common_scripts/check/do_meta.groovy index 7bfe2d1..4b8a89e 100644 --- a/autobuild/Common_scripts/check/do_meta.groovy +++ b/autobuild/Common_scripts/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = entry.meta.componentversionuniqueidentifier() - def offsetpath = entry.offsetpath + def name = entry.metainfo.componentversionuniqueidentifier() + def offsetpath = entry.accumulatedpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } @@ -65,7 +65,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo2 = componentReleaseInfo[key] - def offsetpath = buildinfo2.offsetpath + def offsetpath = buildinfo2.accumulatedpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } diff --git a/autobuild/Common_scripts/checksum/utils.groovy b/autobuild/Common_scripts/checksum/utils.groovy index c2503cd..be7569d 100644 --- a/autobuild/Common_scripts/checksum/utils.groovy +++ b/autobuild/Common_scripts/checksum/utils.groovy @@ -5,6 +5,7 @@ def generatelist(meta, checksummable) { def checksumset = meta.checksuminfo.addChecksumSet(filepath) def files = findFiles(glob: filepath) for (file in files) { + echo "Checksumming: ${file.path}" def sum = sha1(file: file.path).trim() checksumset.addChecksumPair(file.path, sum) } diff --git a/autobuild/Common_scripts/commandline/commandline.groovy b/autobuild/Common_scripts/commandline/commandline.groovy index df99a75..38f80fa 100644 --- a/autobuild/Common_scripts/commandline/commandline.groovy +++ b/autobuild/Common_scripts/commandline/commandline.groovy @@ -2,8 +2,7 @@ String getCommandOutput(String cmd) { if (isUnix()) { return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() } else { - return bat(returnStdout: true, script: """ - @echo off + return powershell(returnStdout: true, script: """ ${cmd} """ ).trim() diff --git a/autobuild/Common_scripts/configuration/configuration.groovy b/autobuild/Common_scripts/configuration/configuration.groovy index 2a7f1ab..7a89975 100644 --- a/autobuild/Common_scripts/configuration/configuration.groovy +++ b/autobuild/Common_scripts/configuration/configuration.groovy @@ -1,7 +1,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) + return meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) } def registerPipelineScriptConfiguration(meta) { @@ -12,7 +12,7 @@ def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, St def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) + return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this diff --git a/autobuild/Common_scripts/configuration/tools.groovy b/autobuild/Common_scripts/configuration/tools.groovy new file mode 100644 index 0000000..dbaeedb --- /dev/null +++ b/autobuild/Common_scripts/configuration/tools.groovy @@ -0,0 +1,59 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +class ToolOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + String m_toolName + + String m_gitURL + String m_revision + String m_repoRoot + int m_cloneDepth + boolean m_shallowClone + + ToolOptions(script) { + m_script = script + m_gitURL = '-' + m_revision = '-' + m_repoRoot = '' + m_cloneDepth = 1 + m_shallowClone = true + } + + void toolName(String name) { + m_toolName = name.trim() + } + + void toolRepo(String gitURL, String revision) { + m_gitURL = gitURL.trim() + m_revision = revision.trim() + } + + void repoRoot(String root) { + m_repoRoot = root.trim() + } + + void clonedepth(int depth) { + m_cloneDepth = depth + } + + void shallowclone(boolean shallow) { + m_shallowClone = shallow + } +} + +ToolOptions getNewToolOptions() { + return new ToolOptions(this) +} + +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) +} + +def getTool(meta, ToolOptions options, String targetDir) { + gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) + + registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/autobuild/Common_scripts/copy/all.groovy b/autobuild/Common_scripts/copy/all.groovy index b7c4607..24aacfb 100644 --- a/autobuild/Common_scripts/copy/all.groovy +++ b/autobuild/Common_scripts/copy/all.groovy @@ -1,29 +1,22 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { script { + // Force workspace to be cleaned + CLEAN_WORKSPACE = "true" meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) - } - } - } - stage('Clean exports') { - steps { - script { - IMPORTS = 'imports' - cleanfolders = [ - IMPORTS, - ] + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") } } } stage('Import source artifacts') { steps { script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def copyoptions = importer.getNewImportOptions() copyoptions.source( SOURCE_ARTIFACT_LOCATION, @@ -33,38 +26,26 @@ pipeline { SOURCE_ARTIFACT_RELEASE_REVISION) echo "Nested: ${NESTED_COPY}" copyoptions.setnested(NESTED_COPY == 'true') - importer.artifactimport(meta, copyoptions, IMPORTS) - } - } - } - stage('Copy source artifacts') { - steps { - script { - dir(IMPORTS) { - fileOperations([ - fileCopyOperation(excludes: '', - flattenFiles: false, - includes: '**/*', - targetLocation: WORKSPACE) - ]) - } + importer.artifactimport(meta, copyoptions, "${WORKSPACE}/.") } } } stage('Archiving artifacts') { steps { script { - def includes = '**/*' - def excludes = "${IMPORTS}/*" - echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" - // This does not work: archiveArtifacts artifacts: includes, excludes: excludes - // so - cleanfolders = [ - IMPORTS, - 'autobuild', - ] - cleaner.clean(cleanfolders) - archiveArtifacts artifacts: includes + // Save the imported meta - while the unwanted meta is written to file then deleted + def stashname = 'Imported meta stash' + stash(allowEmpty: true, + includes: 'meta/**', + name: stashname, + useDefaultExcludes: false) + + def artifactlist = meta.generateartifacts(['**/*']) + cleaner.clean(['meta', 'autobuild']) + + unstash(name: stashname) + + artifactarchiver.archive(meta, artifactlist) } } } @@ -83,4 +64,4 @@ pipeline { echo 'Succeeded build' } } -} \ No newline at end of file +} diff --git a/autobuild/Common_scripts/copy/esw.groovy b/autobuild/Common_scripts/copy/esw.groovy index 2a63171..b9eb737 100644 --- a/autobuild/Common_scripts/copy/esw.groovy +++ b/autobuild/Common_scripts/copy/esw.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -28,7 +28,10 @@ pipeline { stage('Copying embedded software artifacts') { steps { script { - importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST) + def eswimportoptions = importer.getNewImportOptions() + eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) + eswimportoptions.setincrementversionnumber(true) + importer.artifactimport(meta, eswimportoptions, ESWDEST) } } } @@ -41,7 +44,8 @@ pipeline { rm -v -rf meta mv -v "${ESW}"/* . """ - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/autobuild/Common_scripts/copy/filtered.groovy b/autobuild/Common_scripts/copy/filtered.groovy index feaa2ab..3ad24cd 100644 --- a/autobuild/Common_scripts/copy/filtered.groovy +++ b/autobuild/Common_scripts/copy/filtered.groovy @@ -27,7 +27,9 @@ pipeline { stage('Import artifacts') { steps { script { - importer.artifactimport(meta, ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION, IMPORT) + def importoptions = importer.getNewImportOptions() + importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) + importer.artifactimport(meta, importoptions, IMPORT) } } } @@ -48,7 +50,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["filtered/**"]) - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/autobuild/Common_scripts/copy/logic.groovy b/autobuild/Common_scripts/copy/logic.groovy index 7ae0fe1..db9dbc3 100644 --- a/autobuild/Common_scripts/copy/logic.groovy +++ b/autobuild/Common_scripts/copy/logic.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -13,21 +13,27 @@ pipeline { stage('Clean exports') { steps { script { - env.LOGICDEST = "${WORKSPACE}/logic" + LOGICDEST = 'logic' cleanfolders = [ LOGICDEST, "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/el", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Copying logic artifacts') { + stage('Import source artifacts') { steps { script { def logicoptions = importer.getNewImportOptions() - logicoptions.source(LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_ARTIFACT_RELEASE_REPO, LOGIC_ARTIFACT_RELEASE_REVISION) + logicoptions.source( + LOGIC_ARTIFACT_LOCATION, + LOGIC_ARTIFACT_PROJECT, + LOGIC_ARTIFACT_BUILD_SELECTOR, + LOGIC_ARTIFACT_RELEASE_REPO, + LOGIC_ARTIFACT_RELEASE_REVISION) importer.artifactimport(meta, logicoptions, LOGICDEST) } } @@ -35,14 +41,23 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["embedded_logic/**"]) - sh """ - rm -v -rf meta - # The following is removing a deprecated folder - rm -v -rf diff - mv -v "logic"/* . - """ - archiveArtifacts "${artifactlist}" + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + + dir(LOGICDEST) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + + cleaner.clean([LOGICDEST]) + + artifactarchiver.archive(meta, artifactlist) } } } @@ -61,4 +76,4 @@ pipeline { echo 'Succeeded build' } } -} \ No newline at end of file +} diff --git a/autobuild/Common_scripts/differences/dometahierarchy.groovy b/autobuild/Common_scripts/differences/dometahierarchy.groovy index 6280960..3b25798 100644 --- a/autobuild/Common_scripts/differences/dometahierarchy.groovy +++ b/autobuild/Common_scripts/differences/dometahierarchy.groovy @@ -95,8 +95,8 @@ def showmetaitems(metaitems) { String info = '' for (key in metaitems.keySet()) { def relatedmetainfo = metaitems[key] - def offsetpath = relatedmetainfo.offsetpath - info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" + def accumulatedpath = relatedmetainfo.accumulatedpath + info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" } echo info } @@ -768,8 +768,8 @@ def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" - def earlymeta = earlymetaitems[key].meta - def latermeta = latermetaitems[key].meta + def earlymeta = earlymetaitems[key].metainfo + def latermeta = latermetaitems[key].metainfo summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") @@ -803,8 +803,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin def earlymeta = meta.importrelated(true, earlymetafolder) def latermeta = meta.importrelated(true, latermetafolder) - def earlymetaitems = earlymeta.releasewalkerunique(md) - def latermetaitems = latermeta.releasewalkerunique(md) + def earlymetaitems = earlymeta.releasewalkerunique(md, '', earlymetafolder) + def latermetaitems = latermeta.releasewalkerunique(md, '', latermetafolder) echo "Early items:" showmetaitems(earlymetaitems) diff --git a/autobuild/Common_scripts/differences/file.groovy b/autobuild/Common_scripts/differences/file.groovy index ab4102f..ee1b8b3 100644 --- a/autobuild/Common_scripts/differences/file.groovy +++ b/autobuild/Common_scripts/differences/file.groovy @@ -1,15 +1,23 @@ def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { + echo "Comparing file: ${importedfile} with: ${currentfile} option: ${ignorewhitespace}" + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } if (isUnix()) { - def wsparameter = "-w" - if (!ignorewhitespace) { - wsparameter = "" - } sh """ diff -c ${wsparameter} "${importedfile}" "${currentfile}" """ } else { - error "Windows not currently supporting file comparison" + // diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # where diff + # if errorlevel 1 EXIT /b 1 + + diff.exe -c ${wsparameter} "${importedfile}" "${currentfile}" + """ } } diff --git a/autobuild/Common_scripts/differences/json.groovy b/autobuild/Common_scripts/differences/json.groovy index 5ac37f2..dbf47b8 100644 --- a/autobuild/Common_scripts/differences/json.groovy +++ b/autobuild/Common_scripts/differences/json.groovy @@ -1,4 +1,5 @@ def comparefiles(importedjsonfile, currentjsonfile) { + echo "Comparing json file: ${importedjsonfile} with: ${currentjsonfile}" if (isUnix()) { sh """ which jq @@ -8,7 +9,42 @@ def comparefiles(importedjsonfile, currentjsonfile) { """ } else { - error "Windows not currently supporting json file comparison" + // Jq and diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # set + # echo %PATH% + try { + if (Get-Command jq) {} + } + catch { + echo "jq is unavailable" + exit \$false + } + try { + if (Get-Command diff) {} + } + catch { + echo "diff is unavailable" + exit \$false + } + + # echo Sorting json in files to compare + # jq -S sorts json output + jq -S . "${importedjsonfile}" > temp1.txt + if (\$lastexitcode -ge 1) { exit 1 } + jq -S . "${currentjsonfile}" > temp2.txt + if (\$lastexitcode -ge 1) { exit 1 } + + # echo Diffing sorted json files + diff.exe -c temp1.txt temp2.txt + \$lec = \$lastexitcode + + Remove-Item temp1.txt + Remove-Item temp2.txt + + if (\$lec -ge 1) { exit 1 } + """ } } diff --git a/autobuild/Common_scripts/differences/metahierarchy.groovy b/autobuild/Common_scripts/differences/metahierarchy.groovy index 4bcdcdc..01c1890 100644 --- a/autobuild/Common_scripts/differences/metahierarchy.groovy +++ b/autobuild/Common_scripts/differences/metahierarchy.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -37,8 +37,18 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') - importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') + + def earlyimportoptions = importer.getNewImportOptions() + earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) + earlyimportoptions.setincrementversionnumber(false) + earlyimportoptions.setfilter('meta/**') + importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) + + def laterimportoptions = importer.getNewImportOptions() + laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) + laterimportoptions.setincrementversionnumber(false) + laterimportoptions.setfilter('meta/**') + importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) } } } @@ -58,7 +68,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,reports/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,reports/**,${artifactlist}") } } } diff --git a/autobuild/Common_scripts/documentation/doxygen_deploy.groovy b/autobuild/Common_scripts/documentation/doxygen_deploy.groovy index 6642256..5054c02 100644 --- a/autobuild/Common_scripts/documentation/doxygen_deploy.groovy +++ b/autobuild/Common_scripts/documentation/doxygen_deploy.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/autobuild/Common_scripts/hardware/sominfo.groovy b/autobuild/Common_scripts/hardware/sominfo.groovy new file mode 100644 index 0000000..72a3ca9 --- /dev/null +++ b/autobuild/Common_scripts/hardware/sominfo.groovy @@ -0,0 +1,80 @@ +def sominfostore = null + +/* Example expected JSON +{ + "SomType": { + "Som_BA4": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", + "Release repo revision": "" + } + }, + "Som_FC5": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", + "Release repo revision": "" + } + } + } +} +*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: sominfostore, pretty: 1) +} + +def boolean exists() { + return (sominfostore != null) +} + +def set(String jsontext) { + sominfostore = readJSON(text: jsontext) + echo "Som support info:\n${asJsonText()}" +} + +def get() { + return sominfostore +} + +def somtypes() { + return sominfostore['SomType'] +} + +def artifactinfo(somtype) { + return somtype['Artifact info'] +} + +def artifactlocation(artifactinfo) { + return artifactinfo['Location'] +} + +def boolean isjoblocation(artifactinfo) { + return (artifactinfo['Location'] == 'Jenkins job artifact' || + artifactinfo['Location'] == 'Jenkins job artefact') +} + +def boolean isreleaserepolocation(artifactinfo) { + return artifactinfo['Location'] == 'Release repository' +} + +def artifactjob(artifactinfo) { + return [ + "name": artifactinfo['Job'], + "selector": "${artifactinfo['Job selector']}" + ] +} + +def releaserepo(artifactinfo) { + return [ + "repo": artifactinfo['Release repo'], + "revision": artifactinfo['Release repo revision'] + ] +} + +return this \ No newline at end of file diff --git a/autobuild/Common_scripts/import/importer.groovy b/autobuild/Common_scripts/import/importer.groovy index 8624421..fa32ebf 100644 --- a/autobuild/Common_scripts/import/importer.groovy +++ b/autobuild/Common_scripts/import/importer.groovy @@ -20,6 +20,8 @@ class ImportOptions { String m_repo String m_revision + + boolean m_showdetails ImportOptions(script, String sourceselect = 'Jenkins job artifact', @@ -31,7 +33,8 @@ class ImportOptions { boolean flatten = false, boolean includemetainfilter = true, boolean clean = true, - appendtoparentmeta = true) { + appendtoparentmeta = true, + showdetails = false) { m_script = script m_sourceselect = sourceselect.trim() m_incrementversionnumber = incrementversionnumber @@ -43,6 +46,25 @@ class ImportOptions { m_includemetainfilter = includemetainfilter m_clean = clean m_appendtoparentmeta = appendtoparentmeta + m_showdetails = showdetails + } + + String show() { + return """m_incrementversionnumber: ${m_incrementversionnumber} + m_filter: ${m_filter} + m_excludes: ${m_excludes} + m_fingerprint: ${m_fingerprint} + m_nested: ${m_nested} + m_flatten: ${m_flatten} + m_includemetainfilter: ${m_includemetainfilter} + m_clean: ${m_clean} + m_appendtoparentmeta: ${m_appendtoparentmeta} + m_sourceselect: ${m_sourceselect} + m_jobname: ${m_jobname} + m_buildnumberorselector: ${m_buildnumberorselector} + m_repo: ${m_repo} + m_revision: ${m_revision} + """ } void setsourceselect(String sourceselect) { @@ -56,7 +78,7 @@ class ImportOptions { m_sourceselect = sourceselect break default: - error("Unknown type of artifact source select: ${m_sourceselect}") + m_script.error("Unknown type of artifact source select: ${sourceselect}") } } @@ -98,7 +120,19 @@ class ImportOptions { void jenkinsjob(String jobname, String buildnumberorselector) { m_jobname = jobname.trim() - m_buildnumberorselector = buildnumberorselector.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: buildnumberorselector + ] + setsourceselect('Jenkins job artifact') + } + + void jenkinsjob(String jobname, int buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: "${buildnumberorselector}" + ] setsourceselect('Jenkins job artifact') } @@ -113,6 +147,10 @@ class ImportOptions { releaserepo(repo, revision) setsourceselect(sourceselect) } + + void showdetails(boolean showdetails) { + m_showdetails = showdetails + } } ImportOptions getNewImportOptions() { @@ -139,7 +177,7 @@ def conditionalClean(ImportOptions importoptions, String destpath) { } } -def artifactimportfrombuildjob(meta, importoptions, String destpath) { +def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_sourceselect != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") return @@ -147,15 +185,14 @@ def artifactimportfrombuildjob(meta, importoptions, String destpath) { conditionalClean(importoptions, destpath) + echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" + echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" copyArtifacts( filter: formFilter(importoptions), excludes: importoptions.m_excludes, fingerprintArtifacts: importoptions.m_fingerprint, projectName: importoptions.m_jobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: importoptions.m_buildnumberorselector - ], + selector: importoptions.m_buildnumberorselector, flatten: importoptions.m_flatten, target: destpath ) @@ -172,7 +209,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { +def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_sourceselect != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -188,24 +225,31 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) - - relatedimportinfo = importedmeta.getassociated() - echo "Cloning associated repos" - for(relatedmetainfo in relatedimportinfo) { - def relatedmeta = relatedmetainfo.meta - def relatedoffsetpath = relatedmetainfo.offsetpath + + echo "Getting associated metadata from path: ${destpath}" + def collectedmetalist = importedmeta.getassociated(destpath) + for(def collectedmeta in collectedmetalist) { + echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}" + } + + echo "Cloning associated repos inside path: ${destpath}" + for(def collectedmeta in collectedmetalist) { + def relatedmeta = collectedmeta.metainfo + def relatedoffsetpath = collectedmeta.accumulatedpath def cloneurl = relatedmeta.releaseinfo.releaserepoURL def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" - echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - - gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) - - gh.stripgithistory(clonepath) - - def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" - echo "Marking related meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(relatedmeta, metafilepath) + if (relatedmeta.componentidentifier() != '_' && + cloneurl != '') { + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } else { + echo "Not importing component that would be placed at path: ${clonepath}" + } } gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" @@ -218,7 +262,7 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { return importedmeta } -def artifactimportfromrepo(meta, importoptions, String destpath) { +def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_sourceselect != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -246,7 +290,10 @@ def artifactimportfromrepo(meta, importoptions, String destpath) { return importedmeta } -def artifactimport(meta, importoptions, String destpath) { +def artifactimport(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_showdetails) { + echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" + } switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': @@ -276,7 +323,7 @@ def artifactimport(meta, boolean fingerprint = false, boolean nested = false) { echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) importoptions.setincrementversionnumber(incrementversionnumber) @@ -298,7 +345,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false) { echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(buildjob, " ${buildnumber}") importoptions.setincrementversionnumber(incrementversionnumber) @@ -319,7 +366,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false) { echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(buildjob, buildnumberorselector) importoptions.setincrementversionnumber(incrementversionnumber) @@ -330,4 +377,32 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } +// Import the artifacts of a named child component found below parent meta's hierarchy +def artifactimport(meta, + parentmeta, + String parentmetapath, + String childproduct, + String childsubcomponent, + ImportOptions importoptions, + String destinationfolder) { + // Import from parent meta + def childmeta = meta.getNewMetaObject() + def childmeta2 + echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" + echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" + def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) + if (findresults.matchfound) { + childmeta = findresults.matchedmetaitem.metainfo + if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) { + importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) + } else { + importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) + } + childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) + } else { + error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") + } + return childmeta2 +} + return this; diff --git a/autobuild/Common_scripts/install/installhelper.groovy b/autobuild/Common_scripts/install/installhelper.groovy index 96a8b2e..8dbf23e 100644 --- a/autobuild/Common_scripts/install/installhelper.groovy +++ b/autobuild/Common_scripts/install/installhelper.groovy @@ -23,17 +23,32 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { artifactFetched = [:] + if (showdetails) { + echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") + } def importoptions = importer.getNewImportOptions() + importoptions.showdetails(showdetails) for (installer in toBuild) { - def artifactKeys = installerDetails.get(installer)[1] + def installerinfo = installerDetails.get(installer) + if (showdetails) { + echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") + } + def artifactKeys = installerinfo[1] if (artifactKeys instanceof java.lang.String) { artifactKeys = [ artifactKeys ] } + if (showdetails) { + echo ("artifactKeys:\n${artifactKeys}") + } for (artifactKey in artifactKeys) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + artifactDetail = artifactDetails.get(artifactKey) + if (showdetails) { + echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") + } + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail if (artifactFetched.get(artifactKey) == null) { importoptions.jenkinsjob(sourceProject, sourceselector) importoptions.setincrementversionnumber(incrementversionnumber) @@ -45,6 +60,9 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { } } } + if (showdetails) { + echo ("artifactFetched:\n${artifactFetched}") + } return artifactFetched } @@ -93,6 +111,7 @@ def checkISMfilebackend(String xml) { def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) + //echo "ISM file xml: ${xml}" (exitcode, message) = checkISMfilebackend(xml) echo message if (exitcode > 0) { diff --git a/autobuild/Common_scripts/meta/folder.groovy b/autobuild/Common_scripts/meta/folder.groovy index a2fd4e1..6715145 100644 --- a/autobuild/Common_scripts/meta/folder.groovy +++ b/autobuild/Common_scripts/meta/folder.groovy @@ -20,15 +20,19 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - String productname - String subcomponentname - String variant + private String format + private String productname + private String subcomponentname + private String variant + private String uniqueid - ProductInfo(script, productname = "", subcomponentname = "", variant = "") { + ProductInfo(script, String productname = '', String subcomponentname = '', String variant = '') { super(script) + this.format = '0.1' this.productname = productname this.subcomponentname = subcomponentname this.variant = variant + this.uniqueid = UUID.randomUUID().toString() } void set(String productname, String subcomponentname, String variant = '') { @@ -38,7 +42,7 @@ class ProductInfo extends GISJsonSerialised { pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - pstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${productname}\" > NUL 2> NUL", , returnStatus: true) + pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) } if (pstatus > 0) { script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") @@ -48,7 +52,7 @@ class ProductInfo extends GISJsonSerialised { sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - sstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${subcomponentname}\" > NUL 2> NUL", , returnStatus: true) + sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) } if (sstatus > 0) { script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") @@ -59,7 +63,7 @@ class ProductInfo extends GISJsonSerialised { vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - vstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${variant}\" > NUL 2> NUL", , returnStatus: true) + vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) } if (vstatus > 0) { script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") @@ -70,31 +74,44 @@ class ProductInfo extends GISJsonSerialised { this.variant = variant } + void set(ProductInfo otherinfo) { + this.productname = otherinfo.productname + this.subcomponentname = otherinfo.subcomponentname + this.variant = otherinfo.variant + this.uniqueid = otherinfo.uniqueid + } + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] - variant = rawmeta['variant'] ?: "" + variant = rawmeta['variant'] ?: '' + uniqueid = rawmeta['uniqueid'] ?: uniqueid } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'productname' : productname, 'subcomponentname' : subcomponentname, 'variant' : variant, + 'uniqueid': uniqueid, ] return data } } class GitInfo extends GISJsonSerialised { + private String format private String sourcepath private String sourcerevision private String gitrevision private String remoteurl private String remoteinfo - GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + GitInfo(script, sourcepath = '', sourcerevision = '', gitrevision = '', remoteurl = '', remoteinfo = '') { super(script) + this.format = '0.1' this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -102,6 +119,15 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + GitInfo(GitInfo another) { + super(another.script) + this.sourcepath = another.sourcepath + this.sourcerevision = another.sourcerevision + this.gitrevision = another.gitrevision + this.remoteurl = another.remoteurl + this.remoteinfo = another.remoteinfo + } + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { this.sourcepath = sourcepath this.sourcerevision = sourcerevision @@ -110,7 +136,16 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + void set(GitInfo otherinfo) { + this.sourcepath = otherinfo.sourcepath + this.sourcerevision = otherinfo.sourcerevision + this.gitrevision = otherinfo.gitrevision + this.remoteurl = otherinfo.remoteurl + this.remoteinfo = otherinfo.remoteinfo + } + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] sourcepath = rawmeta['sourcepath'] sourcerevision = rawmeta['sourcerevision'] gitrevision = rawmeta['gitrevision'] @@ -120,6 +155,7 @@ class GitInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, 'gitrevision' : gitrevision, @@ -130,57 +166,142 @@ class GitInfo extends GISJsonSerialised { } } -class JenkinsBuildInfo extends GISJsonSerialised { - String buildjobname - int buildnumber - private String buildmachine - private String builddate - - JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { +class JenkinsBuildParameter extends GISJsonSerialised { + private String name + private String valuetype + private String value + + JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { super(script) - this.buildjobname = buildjobname - this.buildnumber = buildnumber - this.buildmachine = buildmachine - this.builddate = builddate + this.name = name + this.valuetype = valuetype + this.value = value } - - void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { - this.buildjobname = buildjobname - this.buildnumber = buildnumber - this.buildmachine = buildmachine - this.builddate = builddate + + void set(String name, String valuetype, String value) { + this.name = name + this.valuetype = valuetype + this.value = value } void deserialiseJson(rawmeta) { - buildjobname = rawmeta['buildjobname'] - buildnumber = rawmeta['buildnumber'] - buildmachine = rawmeta['buildmachine'] - builddate = rawmeta['builddate'] + name = rawmeta['name'] + valuetype = rawmeta['valuetype'] + value = rawmeta['value'] } LinkedHashMap serialisationdata() { def data = [ + 'name' : name, + 'valuetype' : valuetype, + 'value' : value, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String format + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + private ArrayList parameters + private boolean isfullbuild + + JenkinsBuildInfo(script, String buildjobname = '', int buildnumber = 0, String buildmachine = '', String builddate = '', boolean isfullbuild = true) { + super(script) + this.format = '0.1' + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + this.isfullbuild = isfullbuild + parameters = new ArrayList() + } + + void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + this.isfullbuild = isfullbuild + } + + void set(boolean isfullbuild) { + this.isfullbuild = isfullbuild + } + + boolean isFullBuild() { + return isfullbuild + } + + void addParameter(String name, value) { + def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) + parameters.add(p) + return p + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] + parameters = new ArrayList() + if (rawmeta['isfullbuild'] != null) { + for (rawparameter in rawmeta['parameters']) { + def p = new JenkinsBuildParameter(script) + p.deserialiseJson(rawparameter) + parameters.add(p) + } + } + } + + List parametersserialised() { + def associated = [] + for (p in parameters) { + associated.add(p.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, 'buildmachine' : buildmachine, 'builddate' : builddate, + 'isfullbuild' : isfullbuild, + 'parameters' : parametersserialised(), ] return data } } class BuildStampInfo extends GISJsonSerialised { - String buildstampprefix - int revision - String buildstamp + private String format + private String buildstampprefix + private int revision + private String buildstamp BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) + this.format = '0.1' this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp } + BuildStampInfo(BuildStampInfo another) { + super(another.script) + this.buildstampprefix = another.buildstampprefix + this.revision = another.revision + this.buildstamp = another.buildstamp + } + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision @@ -191,6 +312,20 @@ class BuildStampInfo extends GISJsonSerialised { set(other.buildstampprefix, other.revision, other.buildstamp) } + // Override the buildstamp + void override(String buildstamp) { + this.buildstamp = buildstamp + script.echo("Build stamp overridden to: ${this.buildstamp}") + def s = this.buildstamp.split('\\.') + if (s.size() != 4) { + script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") + } + this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" + script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") + this.revision = s[3].toInteger() + script.echo("Revision overridden to: ${this.revision}") + } + String buildstampcommasep() { def String buildstampcommasep = buildstamp.replace(".", ",") return buildstampcommasep @@ -217,6 +352,7 @@ class BuildStampInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] @@ -224,6 +360,7 @@ class BuildStampInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, @@ -233,18 +370,20 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - String releaserepoURL - ArrayList releaseitems - boolean released + private String format + private String releaserepoURL + private ArrayList releaseitems + private boolean released - ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { + ReleaseInfo(script, String releaserepoURL = '', boolean released = false) { super(script) + this.format = '0.1' this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() this.released = released } - void set(String releaserepoURL = "") { + void set(String releaserepoURL = '') { this.releaserepoURL = releaserepoURL } @@ -266,6 +405,7 @@ class ReleaseInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] releaserepoURL = rawmeta['releaserepoURL'] releaseitems = new ArrayList() for (rawreleaseitem in rawmeta['releaseitems']) { @@ -276,6 +416,7 @@ class ReleaseInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'releaserepoURL' : releaserepoURL, 'releaseitems' : releaseitemssserialised(), 'released' : released @@ -290,7 +431,7 @@ class ConfigurationInfo extends GISJsonSerialised { private String configurationversion private String configurationSHA - ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + ConfigurationInfo(script, configurationname = '', configurationURL = '', configurationversion = '', configurationSHA = '') { super(script) this.configurationname = configurationname this.configurationURL = configurationURL @@ -320,7 +461,7 @@ class ChecksumPair extends GISJsonSerialised { private String filepath private String sum - ChecksumPair(script, String filepath = "", String sum = "") { + ChecksumPair(script, String filepath = '', String sum = '') { super(script) this.filepath = filepath this.sum = sum @@ -341,11 +482,13 @@ class ChecksumPair extends GISJsonSerialised { } class ChecksumSet extends GISJsonSerialised { + private String format private String artifactpath private ArrayList checksumpairs - ChecksumSet(script, artifactpath = "") { + ChecksumSet(script, artifactpath = '') { super(script) + this.format = '0.1' this.artifactpath = artifactpath checksumpairs = new ArrayList() } @@ -365,6 +508,7 @@ class ChecksumSet extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] artifactpath = rawmeta['artifactpath'] checksumpairs = new ArrayList() for (rawchecksumpair in rawmeta['checksumpairs']) { @@ -376,6 +520,7 @@ class ChecksumSet extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'artifactpath' : artifactpath, 'checksumpairs' : checksumpairsserialised(), ] @@ -423,19 +568,23 @@ class ChecksumInfo extends GISJsonSerialised { } class PublishableInfo extends GISJsonSerialised { + private String format private String filepath - PublishableInfo(script, String filepath = "") { + PublishableInfo(script, String filepath = '') { super(script) + this.format = '0.1' this.filepath = filepath } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] filepath = rawmeta['filepath'] } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'filepath' : filepath, ] return data @@ -443,43 +592,105 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private static String FORMAT = "0.1" - private static String METAFOLDERNAME = "meta" - private static String METAFILENAME = "info.json" + private static String METAFOLDERNAME = 'meta' + private static String METAFILENAME = 'info.json' private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String format = '0.2' - class RelatedMetaInfo extends GISJsonSerialised { - String offsetpath - MetaInfo meta - - RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + abstract class RelatedMetaInfo extends GISJsonSerialised { + String format + + RelatedMetaInfo(script, format) { super(script) + this.format = format + } + + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + abstract String getrelatedid() + abstract void markreleased() + } + + class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { + private MetaInfo meta + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_1(script, String offsetpath = '', MetaInfo meta = null, String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.1') this.offsetpath = offsetpath + this.uniqueid = uniqueid this.meta = meta } void deserialiseJson(rawmeta) { offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] ?: uniqueid meta = new MetaInfo(script) meta.deserialiseJson(rawmeta['meta']) } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid, 'meta' : meta.serialisationdata(), ] return data } + + String getrelatedid() { + return uniqueid + } + + void markreleased() { + meta.releaseinfo.markreleased() + } } - private String format = FORMAT - ProductInfo productinfo - GitInfo gitinfo - JenkinsBuildInfo jenkinsbuildinfo - BuildStampInfo buildstampinfo - ReleaseInfo releaseinfo - ChecksumInfo checksuminfo + class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { + private String importpath + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_2(script, String importpath = '', String offsetpath = '', String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.2') + this.importpath = importpath + this.offsetpath = offsetpath + this.uniqueid = uniqueid + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid + ] + return data + } + + String getrelatedid() { + return uniqueid + } + + void markreleased() { + // Do nothing + } + } + + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + private ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo private LinkedHashMap custominfo @@ -499,15 +710,11 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } - List getassociated() { - return associatedmeta - } - void markreleased() { script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") releaseinfo.markreleased() for (associated in associatedmeta) { - associated.meta.releaseinfo.markreleased() + associated.markreleased() } } @@ -529,6 +736,7 @@ class MetaInfo extends GISJsonSerialised { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") configurationinfo.add(ci) + return ci } void addCustomInfo(String customId, String customValue) { @@ -539,6 +747,10 @@ class MetaInfo extends GISJsonSerialised { return custominfo.getAt(customId.trim()) } + def isFullBuild() { + return jenkinsbuildinfo.isFullBuild() + } + List publishableinfoserialised() { def publishable = [] for (publishableitem in publishableinfo) { @@ -556,8 +768,8 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - if (rawmeta['format'] != FORMAT) { - script.error("Unsupported meta format found at ${filepath}") + if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { + script.error("Unsupported meta format found") return } productinfo.deserialiseJson(rawmeta['productinfo']) @@ -578,9 +790,15 @@ class MetaInfo extends GISJsonSerialised { } custominfo = rawmeta['custominfo'] for (rawrelatedmeta in rawmeta['associatedmeta']) { - def rm = new RelatedMetaInfo(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) + if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { + def rm = new RelatedMetaInfo_v0_1(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } else if (rawrelatedmeta['format'] == '0.2') { + def rm = new RelatedMetaInfo_v0_2(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } } } @@ -627,19 +845,32 @@ class MetaInfo extends GISJsonSerialised { def String adjustedpath = relativize(destpath) def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { + script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) + + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } } else { script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") - mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) + addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - - RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) - if (appendtoparentmeta) { - associatedmeta.add(rmi) - } - return mi } + + def copyrelatedmeta() { + script.echo("Copying related input meta data to output meta") + for (relatedmetainfo in associatedmeta) { + def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" + script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") + script.fileOperations([ + script.folderCreateOperation(folderPath: destpath), + script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) + ]) + } + } LinkedHashMap serialisationdata() { def data = [ @@ -662,10 +893,10 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here - ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here - String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" - ) { + def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { def String artifactlist = METAARTIFACTS releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { @@ -678,31 +909,6 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } - - // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java - private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { - if (maxdepth > 0) { - if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) - releasemetacollection.add(rmi) - } - if (pathsofar) { - pathsofar += "/" - } - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + relatedmetainfo.offsetpath - releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) - } - } - return releasemetacollection - } - - ArrayList releasewalker(int maxdepth, String startpath = "") { - ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - return releasemetacollection - } String componentidentifier() { return productinfo.productname + "_" + productinfo.subcomponentname @@ -724,53 +930,247 @@ class MetaInfo extends GISJsonSerialised { return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber } - Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + class CollectedMeta { + private MetaInfo metainfo + private String accumulatedpath + private String accumulatedmetapath + + CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { + this.metainfo = metainfo + this.accumulatedpath = accumulatedpath + this.accumulatedmetapath = accumulatedmetapath + } + } + + class IterationInfo { + private ArrayList metacollection + private Map metamap + private String comment + private int level + private String gatheredpath + private String gatheredmetapath + private boolean limiteddepth + private int maxdepth + + IterationInfo(ArrayList metacollection, + Map metamap, + String comment = '', + int level = 0, + String gatheredpath = '', + String gatheredmetapath = '', + boolean limiteddepth = false, + int maxdepth = 0) { + this.metacollection = metacollection + this.metamap = metamap + this.comment = comment + this.level = level + this.gatheredpath = gatheredpath + this.gatheredmetapath = gatheredmetapath + this.limiteddepth = limiteddepth + this.maxdepth = maxdepth + } + + String show() { + return "Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}" + } + + void add(CollectedMeta item) { + metacollection.add(item) + metamap.put(item.metainfo.productinfo.uniqueid, item) + } + } + + def openchildmeta(IterationInfo ii, RelatedMetaInfo relatedmetainfo) { + def MetaInfo mi = new MetaInfo(script) + + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" + def boolean imported + // If this has already been read in from a file + if (relatedmetainfo.getrelatedid() in ii.metamap) { + //script.echo("Already read in meta associated with uniqueid: ${relatedmetainfo.getrelatedid()} - Using it") + mi = ii.metamap[relatedmetainfo.getrelatedid()].metainfo + imported = true + } else if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + imported = true + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") + imported = false + } + + def childmetainfo = [ + 'imported': imported, + 'nextgatheredpath': nextgatheredpath, + 'nextgatheredmetapath': nextgatheredmetapath, + 'childmeta': mi + ] + return childmetainfo + } + + def walkrelated_v0_1(IterationInfo ii) { + //script.echo("Walk related v0.1: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } else { + script.error("There is non v0.1 format related meta info under v0.1 related meta info") + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + def walkrelated(IterationInfo ii) { + //script.echo("Walk related: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + //script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") + if (relatedmetainfo.format == '0.2') { + def childmetainfo = openchildmeta(ii, relatedmetainfo) + if (childmetainfo.imported) { + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + childmetainfo.nextgatheredpath, + childmetainfo.nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = childmetainfo.childmeta.walkrelated(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + // Walks all layers of associated data only + def allrelatedmeta(String metapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + return walkrelated(new IterationInfo(metacollection, metamap, '', 0, '', metapath)) + } + + ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) + script.echo("Release walker iteration info at start:\n${ii.show()}") + def results = walkrelated(ii) + return results.metacollection + } + + // Walks 1 layer of associated data only + def getassociated(String rootpath) { + return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself) + } + + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { - releasemetacollection.put(uniquestring, relatedmetainfo) + def existingentrypathlength = existingentry.accumulatedpath.split('/').size() + def newentrypathlength = metaitem.accumulatedpath.split('/').size() + if (existingentrypathlength > newentrypathlength) { + releasemetacollection.put(uniquestring, metaitem) } } else { - releasemetacollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, metaitem) } return releasemetacollection } - - Map releasewalkerunique(int maxdepth, String startpath = "") { - def ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - def Map releasemetauniquecollection = new LinkedHashMap() - for (relatedmetainfo in releasemetacollection) { - def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) + Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") + + def Map versionuniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() + versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) } - return releasemetauniquecollection + return versionuniquecollection } - Map releasewalkerversionunique(int maxdepth, String startpath = "") { - ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - def Map releasemetaversionuniquecollection = new LinkedHashMap() - for (relatedmetainfo in releasemetacollection) { - def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) + def Map uniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentidentifier() + uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) } - return releasemetaversionuniquecollection + return uniquecollection } MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } + + def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { + def results = [ + 'matchedmetaitem' : new CollectedMeta(this), + 'matchfound' : false + ] + def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") + for (dependentmetaitem in walkresults.metacollection) { + script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") + if (dependentmetaitem.metainfo.productinfo.productname == productname && + dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { + script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") + results.matchedmetaitem = dependentmetaitem + results.matchfound = true + break + } + } + return results + } } MetaInfo metareset(script) { - echo "Resetting meta folder" def mi = new MetaInfo(script) + echo "Resetting meta folder" dir(mi.folder()) { deleteDir() } @@ -778,13 +1178,13 @@ MetaInfo metareset(script) { return mi } -MetaInfo metafromfolder(String folder = "") { +MetaInfo metafromfolder(String folder = '') { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") return buildmeta } -MetaInfo metafromfolderSlurper(String folder = "") { +MetaInfo metafromfolderSlurper(String folder = '') { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") return buildmeta diff --git a/autobuild/Common_scripts/packaging/debian.groovy b/autobuild/Common_scripts/packaging/debian.groovy index ead24fa..9f2f4d3 100644 --- a/autobuild/Common_scripts/packaging/debian.groovy +++ b/autobuild/Common_scripts/packaging/debian.groovy @@ -20,6 +20,7 @@ class DebRootDefinition { String GIS_OPT_KERNEL String GIS_OPT_LOGIC String GIS_OPT_PRINTHEAD + String GIS_OPT_BOOT String USR_BIN String USR_LIB_ARCH @@ -27,7 +28,7 @@ class DebRootDefinition { String HTTP_DEV_DIV String LIB_SYSTEMD - DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp) { + DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix) { ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix @@ -38,69 +39,72 @@ class DebRootDefinition { PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" - ETC = "${DEBPATH}/etc" - GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" - GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" - GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" - GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" - GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" - GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" - GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" - GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" - GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" - GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + ETC = "${DEBPATH}/etc" + inpackagepathsuffix + GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + inpackagepathsuffix + GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + inpackagepathsuffix + GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + inpackagepathsuffix + GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + inpackagepathsuffix + GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" + inpackagepathsuffix + GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + inpackagepathsuffix + GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + inpackagepathsuffix + GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" + inpackagepathsuffix + GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + inpackagepathsuffix + GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + inpackagepathsuffix + GIS_OPT_BOOT = "${DEBPATH}/opt/gis/boot" + inpackagepathsuffix - USR_BIN = "${DEBPATH}/usr/bin" - USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" - LIB_FIRMWARE = "${DEBPATH}/lib/firmware" - HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" - LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + USR_BIN = "${DEBPATH}/usr/bin" + inpackagepathsuffix + USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + inpackagepathsuffix + LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + inpackagepathsuffix + HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + inpackagepathsuffix + LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + inpackagepathsuffix } } -def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp) { - return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp) +def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix = '') { + return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) } -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { - try { - sh """#!/bin/bash - # Exit if any command fails - set -xe +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { + withCredentials([usernamePassword(credentialsId: credentials_id, + usernameVariable: 'SUDO_USERNAME', + passwordVariable: 'SUDO_PASSWORD')]) { + def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" + def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" + try { + sh """#!/bin/bash + # Exit if any command fails + set -xe - echo "${password}" | sudo -S echo + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" - sudo chown -v -R root: "${packagedefs.DEBPATH}" + mkdir -v -p "${packageout}" + echo "Building package:" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" - mkdir -v -p "${packageout}" - echo "Building package:" - dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + mkdir -v -p "${uniquepackageout}" + cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" - mkdir -v -p "${uniquepackageout}" - cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" + echo "Showing package:" + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package contents:" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package:" - dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package contents:" - dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package lintian output:" + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - echo "Showing package lintian output:" - lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + """ + } + catch (Exception e) { + echo "Exception occurred" + sh """#!/bin/bash -v + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" - """ - } - catch (Exception e) { - echo "Exception occurred" - sh """#!/bin/bash -v - echo "${password}" | sudo -S echo - - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" - - exit 1 - """ + exit 1 + """ + } } } diff --git a/autobuild/Common_scripts/publish/do.groovy b/autobuild/Common_scripts/publish/do.groovy index 1b227f4..ea090f0 100644 --- a/autobuild/Common_scripts/publish/do.groovy +++ b/autobuild/Common_scripts/publish/do.groovy @@ -60,7 +60,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { summaryappend("Artifacts: ") - def releasemeta = releaseinfo.meta + def releasemeta = releaseinfo.metainfo def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo @@ -113,7 +113,7 @@ def copydirectory(File dirFrom, File dirTo) { } def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { - def releasemeta = releaseinfo.meta + def releasemeta = releaseinfo.metainfo def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") @@ -140,14 +140,14 @@ def convertitemstopublish(String itemstopublishstring) { // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## -def publishhierarchy(hostedftpsiteinfo, releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() preconditions() def productline = releasemeta.productinfo.productname // Walk the meta information to a max depth, looking for artifacts to publish - def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) echo "Version unique components:" def String versionuniquecomponent = "" @@ -167,8 +167,8 @@ def publishhierarchy(hostedftpsiteinfo, releasemeta, String itemstopublishstring return [summarystring, markunstable, unstablereason] } -def publishhierarchy(hostedftpsiteinfo, releasemeta, String customer, String softwarerelease, String maxdepth) { - return publishhierarchy(hostedftpsiteinfo, releasemeta, "", customer, softwarerelease, maxdepth) +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(hostedftpsiteinfo, releasemeta, releasemetapath, '', customer, softwarerelease, maxdepth) } return this; diff --git a/autobuild/Common_scripts/publish/package.groovy b/autobuild/Common_scripts/publish/package.groovy index 3bd9565..68c41f3 100644 --- a/autobuild/Common_scripts/publish/package.groovy +++ b/autobuild/Common_scripts/publish/package.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -48,7 +48,12 @@ pipeline { steps { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) + + def packageimportoptions = importer.getNewImportOptions() + packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) + packageimportoptions.setincrementversionnumber(true) + packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) + PACKAGE_FOLDER = "${PACKAGEDEST}/package" UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" } diff --git a/autobuild/Common_scripts/publish/promptingpublish.groovy b/autobuild/Common_scripts/publish/promptingpublish.groovy index 4af0d62..ce6a805 100644 --- a/autobuild/Common_scripts/publish/promptingpublish.groovy +++ b/autobuild/Common_scripts/publish/promptingpublish.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -62,6 +62,7 @@ pipeline { echo("Showing filtered releases") for (release in allreleases) { if (!release.toLowerCase().startsWith("superceded") && + !release.toLowerCase().startsWith("superseded") && !release.toLowerCase().startsWith("erroneous") && !release.toLowerCase().startsWith("faulty") && !release.toLowerCase().startsWith("ignore") && @@ -113,6 +114,7 @@ pipeline { 'Select software items', 'Software items', releasemeta, + SOURCEDIR, ITEMSDEFAULTLIST, MAX_META_DEPTH, '20', @@ -134,7 +136,7 @@ pipeline { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, selecteditems, customer, release, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, SOURCEDIR, selecteditems, customer, release, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' @@ -146,7 +148,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/autobuild/Common_scripts/publish/publish.groovy b/autobuild/Common_scripts/publish/publish.groovy index 7beb134..ad33df9 100644 --- a/autobuild/Common_scripts/publish/publish.groovy +++ b/autobuild/Common_scripts/publish/publish.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -58,7 +58,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/autobuild/Common_scripts/publish/publishitems.groovy b/autobuild/Common_scripts/publish/publishitems.groovy index 677970e..9f6c368 100644 --- a/autobuild/Common_scripts/publish/publishitems.groovy +++ b/autobuild/Common_scripts/publish/publishitems.groovy @@ -18,6 +18,7 @@ class PublishableItemsSelection { String name, String description, releasemeta, + String releasemetapath, String preselectedext, String metalevelsstringext, String visibleItemCountExt, @@ -25,7 +26,7 @@ class PublishableItemsSelection { outputSelectionfromPreselect(preselectedmenuitems, preselectedext) def Integer metalevelsext = metalevelsstringext.toInteger() def Integer visibleItemCount = visibleItemCountExt.toInteger() - def metaitems = releasemeta.releasewalkerunique(metalevelsext) + def metaitems = releasemeta.releasewalkerunique(metalevelsext, '', releasemetapath) for (key in metaitems.keySet()) { def releaseinfo = metaitems[key] outputDescription(menuitemdescriptions, releaseinfo) @@ -89,7 +90,7 @@ class PublishableItemsSelection { } def outputDescription(menuitems, releaseinfo) { - def meta = releaseinfo.meta + def meta = releaseinfo.metainfo for (publishableitem in meta.publishableinfo) { menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") @@ -97,7 +98,7 @@ class PublishableItemsSelection { } def outputSelection(menuitems, releaseinfo) { - def meta = releaseinfo.meta + def meta = releaseinfo.metainfo for (publishableitem in meta.publishableinfo) { menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") @@ -113,13 +114,14 @@ PublishableItemsSelection createItemSelector( String name, String description, releasemeta, + String releasemetapath, String preselectedext, String metalevelsstringext, String visibleItemCountExt = '20', String type = 'PT_CHECKBOX') { // Could also be "PT_MULTI_SELECT" PublishableItemsSelection pis = new PublishableItemsSelection(this) - pis.init(name, description, releasemeta, preselectedext, metalevelsstringext, visibleItemCountExt, type) + pis.init(name, description, releasemeta, releasemetapath, preselectedext, metalevelsstringext, visibleItemCountExt, type) return pis } diff --git a/autobuild/Common_scripts/release/do.groovy b/autobuild/Common_scripts/release/do.groovy index 9148572..4dcb091 100644 --- a/autobuild/Common_scripts/release/do.groovy +++ b/autobuild/Common_scripts/release/do.groovy @@ -112,20 +112,20 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") def boolean result = true - for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { + for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { def filterpath = artifactfilter - if (buildinfo.offsetpath) { - filterpath = buildinfo.offsetpath + '/' + artifactfilter + if (buildinfo.accumulatedpath) { + filterpath = buildinfo.accumulatedpath + '/' + artifactfilter } // Copy relevant artifacts to temp folder - echo "Copying artifacts: ${filterpath} to ${tempfolder}" + echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" try { copyArtifacts( filter: filterpath, projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" ], target: tempfolder ) @@ -136,9 +136,9 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } if (result) { // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath dir(originalrootdir) { - sh("mv * ${repofolder}") + sh("mv -v * ${repofolder}") } } summaryappend("${artifactfilter},") @@ -235,6 +235,7 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) summaryappend("Rel Tag ") for (tag in taglist) { if (releasegittag(postcommitshacode, tag)) { + echo "Tag added: ${tag} with sha: ${postcommitshacode}" tagadded = true } } @@ -374,7 +375,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { - def buildmeta = buildinfo.meta + def buildmeta = buildinfo.metainfo def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" @@ -395,13 +396,13 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri repoandtagsappend("\n") } -def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { +def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { def int md = maxdepth.toInteger() resetfolders() preconditions() // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo = buildmeta.releasewalkerunique(md) + def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { @@ -410,17 +411,29 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String echo versioncomponent // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) - echo "Version unique components:" - def String versionuniquecomponent = "" + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) + def String versionuniquecomponent = '' + def String versionwithpathscomponent = '' for (key in componentReleaseInfoVersionUnique.keySet()) { versionuniquecomponent += "${key}\n" + versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" } + echo 'Version unique components:' echo versionuniquecomponent + echo 'Version unique components with paths:' + echo versionwithpathscomponent + if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { - error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" + } + } + + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { + error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" } } @@ -428,8 +441,11 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - if (toplevelid != key) { - releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + echo "Releasing component: ${buildinfo.metainfo.componentidentifier()}" + if (buildinfo.metainfo.componentidentifier() != '___' && buildinfo.metainfo.componentidentifier() != '_') { + if (toplevelid != key) { + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + } } } // Do the top level last diff --git a/autobuild/Common_scripts/release/release.groovy b/autobuild/Common_scripts/release/release.groovy index 4220f3b..82af511 100644 --- a/autobuild/Common_scripts/release/release.groovy +++ b/autobuild/Common_scripts/release/release.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -23,18 +23,16 @@ pipeline { stage('Get build artifact meta') { steps { script { - BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" - copyArtifacts( - filter: 'meta/**', - projectName: BUILD_JOB, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR - ], - target: BUILD_ARTIFACTS - ) - def mfs = autobuilderscript.getmetafolderscript() - buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + BUILD_ARTIFACTS = 'build' + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) + importoptions.setincrementversionnumber(false) + importoptions.setfilter('meta/**') + importoptions.setappendtoparentmeta(false) + buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) } } } @@ -44,7 +42,7 @@ pipeline { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" - (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' @@ -56,7 +54,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/autobuild/Common_scripts/staticanalysis/ccm.groovy b/autobuild/Common_scripts/staticanalysis/ccm.groovy index f0e5b80..ae91cb0 100644 --- a/autobuild/Common_scripts/staticanalysis/ccm.groovy +++ b/autobuild/Common_scripts/staticanalysis/ccm.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -13,12 +13,6 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - PYTHONURL = 'git@hydra:tools/python.git' - PYTHONBRANCH = 'origin/3.6.3_windows' - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - CCMURL = 'git@hydra:tools/ccm.git' CCMBRANCH = 'origin/master' CCMDIR = "${WORKSPACE}/ccm" @@ -30,49 +24,40 @@ pipeline { stage('Checkout source') { steps { script { - SOURCEREVISION = "${SOURCE_REVISION}" SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) } } } stage("Clean exports") { steps { script { - STATICANALYSIS = 'staticanalysis' + RESULTS = 'results' cleanfolders = [ - STATICANALYSIS, + RESULTS, ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } stage('Static analysis') { steps { script { - def CCMOLDER = "${STATICANALYSIS}/ccm" - dir(CCMOLDER) { - echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" + def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" + writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) + echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" + + dir(RESULTS) { + echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" if (isUnix()) { sh """#!/bin/bash - find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \\; - # Uses mono - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + ${CCM}" "${ccmconfigurationfile}" > ccm.xml """ } else { + echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" bat """ - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml """ } } @@ -82,25 +67,6 @@ pipeline { } } } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" - } - } - } } post { always { diff --git a/autobuild/Common_scripts/staticanalysis/cpd.groovy b/autobuild/Common_scripts/staticanalysis/cpd.groovy index 6f3a33f..c04649c 100644 --- a/autobuild/Common_scripts/staticanalysis/cpd.groovy +++ b/autobuild/Common_scripts/staticanalysis/cpd.groovy @@ -129,7 +129,8 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/autobuild/Common_scripts/staticanalysis/cppcheck.groovy b/autobuild/Common_scripts/staticanalysis/cppcheck.groovy index bdc6a2a..d959770 100644 --- a/autobuild/Common_scripts/staticanalysis/cppcheck.groovy +++ b/autobuild/Common_scripts/staticanalysis/cppcheck.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -91,7 +91,8 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/autobuild/Common_scripts/test/testethernetproduct.groovy b/autobuild/Common_scripts/test/testethernetproduct.groovy new file mode 100644 index 0000000..5e2ce53 --- /dev/null +++ b/autobuild/Common_scripts/test/testethernetproduct.groovy @@ -0,0 +1,204 @@ +import jenkins.model.Jenkins + +pipeline { + agent { label AGENT_LABEL } + stages { + stage ("Initial Setup") { + steps{ + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage ("Checkout test environment") { + steps { + script { + toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") + + def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' + def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' + def TEST_ENVIRONMENT_REVISION = 'origin/master' + + def environmentoptions = toolsimporter.getNewToolOptions() + environmentoptions.toolName('Ethernet product test environment') + environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) + + TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" + def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") + TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] + echo "Selected environment:\n${TEST_ENVIRONMENT}" + } + } + } + stage ('Setup') { + agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } + stages { + stage ("Checkout dependencies") { + steps { + script { + def PYTHON_URL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHON_REVISION = 'origin/3.6.3_linux' + } + else { + PYTHON_REVISION = 'origin/3.8.10_windows_for_test' + } + PYTHONDIR = "${WORKSPACE}/python" + + def pythonimportoptions = toolsimporter.getNewToolOptions() + pythonimportoptions.toolName('Python') + pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) + toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + + def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' + TEST_SUITE_DIR = 'TestSuiteDir' + env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" + + def regressiontestimportoptions = toolsimporter.getNewToolOptions() + regressiontestimportoptions.toolName('Tests') + regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) + + env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" + + def resultimportoptions = toolsimporter.getNewToolOptions() + resultimportoptions.toolName('Test history') + resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) + toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) + + def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' + env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" + + def coreimportoptions = toolsimporter.getNewToolOptions() + coreimportoptions.toolName('Test core') + coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) + toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) + + def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' + def FTDI_SOURCE_REVISION = 'origin/Core-98' + FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" + + def ftdiimportoptions = toolsimporter.getNewToolOptions() + ftdiimportoptions.toolName('FTDI relay driver') + ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) + toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) + + def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' + HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" + + def resultprocessingoptions = toolsimporter.getNewToolOptions() + resultprocessingoptions.toolName('History processing') + resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) + toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) + } + } + } + stage ("Import application to test") { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + APPLICATION_UNDER_TEST = 'application_under_test' + env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" + + def applicationimportoptions = importer.getNewImportOptions() + applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) + applicationimportoptions.setfilter('app_package/**,control_package/**') + applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) + + meta.productinfo.set(applicationmeta.productinfo) + meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" + meta.gitinfo.set(applicationmeta.gitinfo) + meta.buildstampinfo.set(applicationmeta.buildstampinfo) + } + } + } + stage("Clean") { + steps { + script { + LOGGING_ROOT = 'Logging' + env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" + cleanfolders = [ + GIS_TEST_OUTPUT_LOGGING_ROOT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage ('Test') { + steps { + script { + lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { + if (POWER_CYCLE_HARDWARE == 'true') { + dir(FTDI_SOURCE_DIR) { + bat """ + "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True + """ + } + } + if (UPDATE_HARDWARE == 'true') { + dir(GIS_TEST_SUITE_CORE) { + bat """ + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" + """ + rc = bat (script:""" + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" + """, + returnStatus: true) + if (rc == 1) { + unstable('FAILED while inspecting state of hardware') + } + } + } + if (DO_TEST == 'true') { + rc = bat(script:""" + ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED while running tests') + } + } + } + } + } + } + stage("Archive Test History") { + steps{ + script { + dir(HISTORY_FOLDER) { + rc = bat (script:""" + "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED to process history') + } + } + } + } + } + stage("Archive test artifacts"){ + steps{ + script { + def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) + } + } + } + } + } + } +} + diff --git a/autobuild/Common_scripts/versioncontrol/git.groovy b/autobuild/Common_scripts/versioncontrol/git.groovy index f5b2ebd..a22054b 100644 --- a/autobuild/Common_scripts/versioncontrol/git.groovy +++ b/autobuild/Common_scripts/versioncontrol/git.groovy @@ -12,7 +12,7 @@ def boolean isShallowClonedRepo(String gitrootpath) { isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() } else { - isshallowclone = bat(returnStdout: true, script: "@git rev-parse --is-shallow-repository").trim().toBoolean() + isshallowclone = powershell(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() } echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" } @@ -89,9 +89,9 @@ def String checkouttoreporoot(String gitURL) { if [ \$? -ne 0 ]; then exit 1; fi """ } else { - bat """ + powershell """ git status - if errorlevel 1 EXIT /b 1 + if (\$lastexitcode -ge 1) { exit 1 } """ } } @@ -109,9 +109,9 @@ def checkgitlfsinstall() { """ } else { - bat """ + powershell """ git lfs install - if errorlevel 1 EXIT /b 1 + if (\$lastexitcode -ge 1) { exit 1 } """ } } @@ -185,6 +185,7 @@ def showsparseitemlist(ArrayList sparseitemlist = new ArrayList( } def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { + echo "Git: Checking out: URL: ${gitURL} Revision: ${revision} to: ${targetDir}" checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -193,7 +194,6 @@ def docheckout(String gitURL, String revision, int cloneDepth, String reporootpa branches: [ [name: revision] ], - doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], @@ -201,6 +201,7 @@ def docheckout(String gitURL, String revision, int cloneDepth, String reporootpa [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'PruneStaleBranch'], ], submoduleCfg: [], ]) @@ -221,7 +222,7 @@ def gitinit(String targetDir, String reporootpath) { git init "${targetDir}" """ } else { - bat """ + powershell """ git init "${targetDir}" """ } @@ -245,7 +246,7 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { git config lfs.fetchinclude "${sparseitemliststring}" """ } else { - bat """ + powershell """ git config lfs.fetchinclude "${sparseitemliststring}" """ } @@ -263,10 +264,10 @@ def gitsparserestore(String targetDir) { git reset --hard > /dev/null """ } else { - bat """ - git rm -rf . 2> nul - git clean -fxd 2> nul - git reset --hard 2> nul + powershell """ + git rm -rf . > \$null + git clean -fxd > \$null + git reset --hard > \$null """ } } @@ -311,8 +312,8 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } else { - result = bat( script: """ - @git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + result = powershell( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY """, returnStatus: true ) diff --git a/autobuild/Common_scripts/versionnumbering/do.groovy b/autobuild/Common_scripts/versionnumbering/do.groovy index 666aee3..208a60f 100644 --- a/autobuild/Common_scripts/versionnumbering/do.groovy +++ b/autobuild/Common_scripts/versionnumbering/do.groovy @@ -4,17 +4,17 @@ def stamp(convertdetails) { def filetoconvert = convertdetail[0] def pattern = convertdetail[1] def replacement = convertdetail[2] - //echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - //echo "Using python: ${PYTHON} and script: ${stampscript}" + echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } else { - bat """ - "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + powershell """ + & "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } } diff --git a/autobuild/Common_scripts/versionnumbering/generate.groovy b/autobuild/Common_scripts/versionnumbering/generate.groovy index 71d7bf6..5e974e2 100644 --- a/autobuild/Common_scripts/versionnumbering/generate.groovy +++ b/autobuild/Common_scripts/versionnumbering/generate.groovy @@ -28,7 +28,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } else { - changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() + changecount = powershell(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" def int mvnp1 = maxversionnumber() + 1 @@ -39,21 +39,21 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() } else { - gitrevision = bat(returnStdout: true, script: "@git rev-parse HEAD").trim() + gitrevision = powershell(returnStdout: true, script: "git rev-parse HEAD").trim() } echo "Revision: ${gitrevision}" if (isUnix()) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() } else { - remoteinfo = bat(returnStdout: true, script: "@git remote show origin").trim() + remoteinfo = powershell(returnStdout: true, script: "git remote show origin").trim() } echo "Remote info: ${remoteinfo}" if (isUnix()) { remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() } else { - remoteurl = bat(returnStdout: true, script: "@git config --get remote.origin.url").trim() + remoteurl = powershell(returnStdout: true, script: "git config --get remote.origin.url").trim() } echo "Remote URL: ${remoteurl}" } @@ -98,6 +98,9 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { def integerstrings = sourcebranch.findAll(/\d+/) def integers = [] for (integerstring in integerstrings) { + if (integerstring.length() > 6) { + integerstring = integerstring.substring(0, 5) + } int integer = integerstring.toInteger() integer = integer % (maxversionnumber() + 1) integers.add(integer) @@ -154,14 +157,22 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { + generate(meta, sourcerevision, sourcepath, basebuildnumber) + meta.buildstampinfo.override(pregeneratedbuildstamp) + + return pregeneratedbuildstamp +} + String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { def int bbn = basebuildnumber.trim().toInteger() return generate(meta, sourcerevision, sourcepath, bbn) } String copyfrom(meta, othermeta) { - meta.gitinfo = othermeta.gitinfo - meta.buildstampinfo = othermeta.buildstampinfo + // Deep copy + meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) + meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) return meta.buildstampinfo.buildstamp } diff --git a/autobuild/synthesis/synthesis.gsh b/autobuild/synthesis/synthesis.gsh index 34618db..dec7fd8 100644 --- a/autobuild/synthesis/synthesis.gsh +++ b/autobuild/synthesis/synthesis.gsh @@ -170,7 +170,7 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["embedded_logic/**"]) + def artifactlist = meta.generateartifacts(["embedded_logic/**","el/firmware/**"]) archiveArtifacts "${artifactlist}" } } diff --git a/autobuild/synthesis/synthesis.py b/autobuild/synthesis/synthesis.py index b0bc653..fba24bb 100644 --- a/autobuild/synthesis/synthesis.py +++ b/autobuild/synthesis/synthesis.py @@ -7,28 +7,31 @@ from distutils.util import strtobool WORKSPACE=os.environ["WORKSPACE"] print("WORKSPACE is {0}".format(WORKSPACE)) -if (("PREVENT_AUTO_VERSIONING" in os.environ) and ('true' == os.environ["PREVENT_AUTO_VERSIONING"])) : - print("NOT performing AUTO Versioning") -else : - from update_fw_ver import updateFwVer - - if os.path.exists("el/firmware/el_generic_ip") : - versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/el_generic_ip/generic_ip/generic_versioning/doc/generic_versioning.json")) +if os.path.exists("el/firmware/common/scripts/create_pkg.sh"): + print("versioning for Non-QSYS products will be handled later in script") +else: + if (("PREVENT_AUTO_VERSIONING" in os.environ) and ('true' == os.environ["PREVENT_AUTO_VERSIONING"])) : + print("NOT performing AUTO Versioning") else : - versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/generic_ip/generic_versioning/doc/generic_versioning.json")) + from update_fw_ver import updateFwVer - try: - buildStamp=os.environ["BUILD_STAMP"] - except: - print("unable to access build_stamp env variable") - sys.exit(1) + if os.path.exists("el/firmware/el_generic_ip") : + versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/el_generic_ip/generic_ip/generic_versioning/doc/generic_versioning.json")) + else : + versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/generic_ip/generic_versioning/doc/generic_versioning.json")) - try: - if (0 != updateFwVer(versioningJsonFileName,buildStamp)) : + try: + buildStamp=os.environ["BUILD_STAMP"] + except: + print("unable to access build_stamp env variable") + sys.exit(1) + + try: + if (0 != updateFwVer(versioningJsonFileName,buildStamp)) : + sys.exit(1) + except: + print("Error updating fw version") sys.exit(1) - except: - print("Error updating fw version") - sys.exit(1) QPF_TOP=os.environ["QPF_TOP"] QSYS_TOP=os.environ["QSYS_TOP"] diff --git a/logic_memory_mapper/gen_htm.py b/logic_memory_mapper/gen_htm.py index 848acb2..e3fe0b8 100644 --- a/logic_memory_mapper/gen_htm.py +++ b/logic_memory_mapper/gen_htm.py @@ -82,7 +82,7 @@ class MemMapperHtm(MemMapperBase): # --------------------------------------------------------- def setWidth(self, width): self.__tableWidth = width - + # --------------------------------------------------------- # Configure the notes width # --------------------------------------------------------- @@ -181,6 +181,9 @@ class MemMapperHtm(MemMapperBase): notesStr += 'rwpSimilar to wp, but also generates a (separate) pulse on a read access to the register\n' notesStr += '\n' notesStr += '\n' + notesStr += 'bitdefDefines the bits used in the read, set, clear registers\n' + notesStr += '\n' + notesStr += '\n' notesStr += '  \n' notesStr += '\n' notesStr += '\n' @@ -463,7 +466,7 @@ class MemMapperHtm(MemMapperBase): regType = regDict[regName][attr.TYPE] else: regType = ' ' - + if attr.SUBTYPE in regDict[regName] : regSubType = regDict[regName][attr.SUBTYPE] else : @@ -485,6 +488,17 @@ class MemMapperHtm(MemMapperBase): # Generate a hex string for the value (with a leading '0x') addrStr = '0x' + format(addrVal, 'x') + rowSpan = 1 + hasFields = attr.FIELDS in regDict[regName] + + if hasFields : + fields = regDict[regName][attr.FIELDS] + rowSpan = len(fields) + 1 + + # Bitdefs are not real registers so mark as N/A + if fields[list(fields)[0]][attr.TYPE] == regtype.BIT_DEFINITION: + addrStr = 'N/A' + # Gather together each column's string into a list for this register entryList = [regNameStr, addrStr, @@ -495,13 +509,6 @@ class MemMapperHtm(MemMapperBase): reset, regDict[regName][attr.DESCRIPTION]] - rowSpan = 1 - hasFields = attr.FIELDS in regDict[regName] - - if hasFields : - fields = regDict[regName][attr.FIELDS] - rowSpan = len(fields) + 1 - # Draw the row for the register self.__drawContents(entryList, wfp, False, rowSpan, not hasFields, lastReg) @@ -520,7 +527,7 @@ class MemMapperHtm(MemMapperBase): resetVal = '0' else : resetVal = fields[fieldName][attr.RESET] - + if attr.SUBTYPE in fields[fieldName] : regSubType = fields[fieldName][attr.SUBTYPE] else : diff --git a/logic_memory_mapper/gen_pkg.py b/logic_memory_mapper/gen_pkg.py index 2570f78..79efff3 100644 --- a/logic_memory_mapper/gen_pkg.py +++ b/logic_memory_mapper/gen_pkg.py @@ -208,9 +208,17 @@ class MemMapperPkg(MemMapperBase) : # Pad the string to match the desired o/p width fieldFullName = MemMapperPkg.__padString(fieldFullName, MemMapperPkg.outputNameWidth) - # Output the bit-slice subtype declaration - wfp.write(' subtype ' + fieldFullName + - ' is natural range ' + topStr + ' downto ' + bottomStr + ';\n') + if fieldLen == 1: + + # Output single bit as a constant + wfp.write(' constant ' + fieldFullName + + ' : integer := ' + topStr + ';\n') + + else: + + # Output the bit-slice subtype declaration + wfp.write(' subtype ' + fieldFullName + + ' is natural range ' + topStr + ' downto ' + bottomStr + ';\n') # Put a blank line between busses wfp.write('\n') diff --git a/logic_memory_mapper/memMapperBase.py b/logic_memory_mapper/memMapperBase.py index 16d3cb1..52425ae 100644 --- a/logic_memory_mapper/memMapperBase.py +++ b/logic_memory_mapper/memMapperBase.py @@ -62,8 +62,9 @@ class regtype : CONSTANT = 'c' RESERVED = 'rsv' READ_WRITE_PULSE = 'rwp' + BIT_DEFINITION = 'bitdef' - typeList = [WRITE, WRITE_PULSE, WRITE_CLR0, READ, READ_PULSE, CONSTANT, RESERVED, READ_WRITE_PULSE] + typeList = [WRITE, WRITE_PULSE, WRITE_CLR0, READ, READ_PULSE, CONSTANT, RESERVED, READ_WRITE_PULSE, BIT_DEFINITION] # --------------------------------------------------------- # Abstract base class to locate JSON descriptions of @@ -81,7 +82,7 @@ class MemMapperBase : # used in validation of model writeTypes = [regtype.WRITE, regtype.WRITE_CLR0, regtype.WRITE_PULSE, regtype.READ_WRITE_PULSE] readTypes = [regtype.READ, regtype.READ_PULSE, regtype.CONSTANT] - neutralTypes = [regtype.RESERVED] + neutralTypes = [regtype.RESERVED, regtype.BIT_DEFINITION] noReadTypes = [regtype.WRITE_CLR0, regtype.RESERVED] wrPulseTypes = [regtype.WRITE_PULSE, regtype.READ_WRITE_PULSE] diff --git a/verification/lib/tests/utils/dataStreamer7Utils.h b/verification/lib/tests/utils/dataStreamer7Utils.h index 67200d1..e3301a0 100644 --- a/verification/lib/tests/utils/dataStreamer7Utils.h +++ b/verification/lib/tests/utils/dataStreamer7Utils.h @@ -12,7 +12,7 @@ class dataStreamer7Utils { public: const uint32_t numCmdsInBlock = 15; - + // Bit offsets of data streamer commands enum DataStreamerCmdOffsets_e { dsCmdOffset = 0, @@ -32,7 +32,7 @@ public: loadCmdQPaddr = 0x4, nop = 0xf }; - + // Valid data streamer destinations. Note cmdQueueDest is only for load command qwueue commands enum headDest_e { headDest0 = 0, @@ -76,6 +76,8 @@ protected: ((dest & 0x0f) << dsDestOffset) | ((loop & 0xff) << dsLoopOffset) | ((param0 & 0xfff) << dsParam0Offset); + if (cmd == rdDataPaddr && (param0 & 0xfff) > 256) + VPrint("***** Error WriteDsCmd rdDataPaddr parameter 0 (numwords) %d > 256 max\n", param0 & 0xfff); } return word; }; diff --git a/verification/lib/tests/utils/fpm6Utils.h b/verification/lib/tests/utils/fpm6Utils.h index 500d637..3a8e1bb 100644 --- a/verification/lib/tests/utils/fpm6Utils.h +++ b/verification/lib/tests/utils/fpm6Utils.h @@ -31,32 +31,39 @@ protected: // ------------------------------------ // Set up FPM for PD emulation // - void configFpmForEmulPulses(CCoreAuto* pCore, + void configFpmForEmulPulses(CCoreAuto* pCore, double encFreqKHz, - double divRatio, - uint32_t numSubPixPhases, - double clkFreqMHz = 100.0, - uint32_t seq[] = NULL, + double divRatio, + uint32_t numSubPixPhases, + double clkFreqMHz = 100.0, + uint32_t seq[] = NULL, uint32_t seqLen = 0, - uint32_t pgSel = 0) + uint32_t pgSel = 0, + CPulseGenerator3Auto* pPulseGenIn = 0) // If pPulseGenIn is valid, it overides any selection from pgSel. + // Used to add > 4 Pulse Generators, where required. { configPdEmulator (pCore, divRatio, encFreqKHz, clkFreqMHz); - fpmConfig (pCore, divRatio, numSubPixPhases, seq, seqLen, pgSel); + fpmConfig (pCore, divRatio, numSubPixPhases, seq, seqLen, pgSel, pPulseGenIn); }; // ------------------------------------ // Configure the FPM's pulse generator // - int fpmConfig(CCoreAuto* pCore, double divRatio, uint32_t numSubPixPhases, uint32_t seq[], uint32_t seqLen, uint32_t pgSel = 0) + int fpmConfig(CCoreAuto* pCore, double divRatio, uint32_t numSubPixPhases, uint32_t seq[], uint32_t seqLen, uint32_t pgSel = 0, CPulseGenerator3Auto* pPulseGenIn = 0) { CPulseGenerator3Auto* pPulseGen; - switch(pgSel) + if (pPulseGenIn != 0) + pPulseGen = pPulseGenIn; + else { - case 0 : pPulseGen = pCore->pPulseGenerator30; break; - case 1 : pPulseGen = pCore->pPulseGenerator31; break; - case 2 : pPulseGen = pCore->pPulseGenerator32; break; - case 3 : pPulseGen = pCore->pPulseGenerator33; break; + switch(pgSel) + { + case 0 : pPulseGen = pCore->pPulseGenerator30; break; + case 1 : pPulseGen = pCore->pPulseGenerator31; break; + case 2 : pPulseGen = pCore->pPulseGenerator32; break; + case 3 : pPulseGen = pCore->pPulseGenerator33; break; + } } // Program the sequence if a buffer pointer is provided and the length isn't zero. @@ -99,7 +106,7 @@ protected: CSR_PULSE_GENERATOR_3_FRACTIONAL_DIVIDER_CONTROL_UDP_MIN_POS_MASK); // enable the ch-ctrl sequencer - pPulseGen->pGenericSequencer15Channel->pCtrl->SetSeqEnable(1); + pPulseGen->pGenericSequencer16Channel->pCtrl->SetSeqEnable(1); // Issue a start_req to the ch-ctrl sequencer diff --git a/verification/lib/tests/utils/tcpClient.cpp b/verification/lib/tests/utils/tcpClient.cpp index 5d274d4..a68a407 100644 --- a/verification/lib/tests/utils/tcpClient.cpp +++ b/verification/lib/tests/utils/tcpClient.cpp @@ -50,7 +50,7 @@ uint32_t tcpClient::ipv4_chksum (uint32_t* buf, uint32_t len, bool debug) { uint32_t sum = 0; - for (int idx = 0; idx < len; idx+=2) + for (int idx = 0; idx < (len - 1); idx+=2) { sum += ((buf[idx] & 0xff) << 8) | (buf[idx+1] & 0xff); @@ -542,4 +542,4 @@ uint32_t tcpClient::processFrame (uint32_t* rx_data, uint32_t rx_len) return error; -} \ No newline at end of file +} diff --git a/verification/tb_template/test/tb_ctrl.vhd b/verification/tb_template/test/tb_ctrl.vhd index a1975e2..eeb4d09 100644 --- a/verification/tb_template/test/tb_ctrl.vhd +++ b/verification/tb_template/test/tb_ctrl.vhd @@ -36,20 +36,20 @@ port ( clk : out std_logic := '1'; clk_div2 : out std_logic := '1'; rst_n : out std_logic := '0'; - + count_vec : out std_logic_vector(31 downto 0); timeout : in std_logic_vector(31 downto 0); - + -- Simulation status and control inputs error : in std_logic; do_stop : in std_logic; do_finish : in std_logic; partial_test : in std_logic; - + -- External end-of-simulation flags failed : out boolean; passed : out boolean - + ); end entity; @@ -57,7 +57,11 @@ architecture sim of tb_ctrl is package tb_pkg is new work.tb_pkg; use tb_pkg.all; - + + signal clk_x2_e2 : std_logic := '1'; + signal clk_x2_e1 : std_logic := '1'; + signal clk_e1 : std_logic := '1'; + signal is_gui_run : boolean := false; signal count : integer := 0; signal end_sim : std_logic; @@ -69,26 +73,31 @@ begin ----------------------------------------- -- Clock generation with concurrent procedure call - clk_gen(clk_x2, Real(CLK_FREQ_MHZ) * 2.0E6); -- x2 clock - + clk_gen(clk_x2_e2, Real(CLK_FREQ_MHZ) * 2.0E6); -- x2 clock + -- Export counter count_vec <= std_logic_vector(to_unsigned(count, 32)); -- Derive system clock from master clock - process(clk_x2) + process(clk_x2_e2) begin - if clk_x2'event and clk_x2 = '1' then - clk <= not clk; + if clk_x2_e2'event and clk_x2_e2 = '1' then + clk_e1 <= not clk_e1; end if; end process; - process(clk) + process(clk_e1) begin - if clk'event and clk = '1' then + if clk_e1'event and clk_e1 = '1' then clk_div2 <= not clk_div2; end if; end process; + -- delta time corrections. All clocks should be in phase as per a PLL + clk_x2_e1 <= clk_x2_e2; + clk_x2 <= clk_x2_e1; + clk <= clk_e1; + -- Generate a reset process (clk) begin @@ -97,14 +106,14 @@ begin rst_n <= '1' when count > 10 else '0'; end if; end process ; - + -- End-of-simulation signals end_sim <= '1' when count = to_integer(unsigned(timeout)) or do_stop = '1' or do_finish = '1' else '0'; failed <= true when end_sim = '1' and (error = '1' or count = to_integer(unsigned(timeout))) else false; passed <= true when end_sim = '1' and error = '0' else false; - + ----------------------------------------- - -- Simulation control from VProc over + -- Simulation control from VProc over -- CSR bus ----------------------------------------- @@ -128,7 +137,7 @@ begin if error = '1' then report "*** FAIL: errors found" severity failure; - + else if count >= to_integer(unsigned(timeout)) then report "*** FAIL: timeout" severity failure; @@ -138,7 +147,7 @@ begin report "SUCCESS" severity note; end if; end if; - + if is_gui_run or (do_stop = '1' and do_finish = '0') then stop(0); else @@ -149,4 +158,4 @@ begin end if; end process; -end architecture; \ No newline at end of file +end architecture;