diff --git a/Common_scripts/build/postactions.groovy b/Common_scripts/build/postactions.groovy index 8b18908..3206004 100644 --- a/Common_scripts/build/postactions.groovy +++ b/Common_scripts/build/postactions.groovy @@ -85,16 +85,20 @@ def getActionJobName(postbuildactions, String action, String alternativeparamnam return jobname } -def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { +def deployjob(meta, String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername, String releaserepoparametername = '') { def jobname = getActionJobName(postbuildactions, action, alternativeparamname) if (jobname) { + def parameters = [ + new StringParameterValue(jobnameparametername, JOB_NAME), + new StringParameterValue(jobselectorparametername, buildselector), + ] + if (releaserepoparametername) { + parameters += new StringParameterValue(releaserepoparametername, meta.releaseinfo.releaserepoURL) + } build( wait: false, job: jobname, - parameters: [ - string(name: jobnameparametername, value: JOB_NAME), - string(name: jobselectorparametername, value: buildselector), - ] + parameters: parameters ) } } @@ -107,7 +111,7 @@ def readPostBuildInfo() { return postbuildinfo } -def performSuccessfulBuildActions() { +def performSuccessfulBuildActions(meta) { def postbuildactions if (params.POST_BUILD_INFO != null) { def postbuildinfo = readPostBuildInfo() @@ -119,12 +123,12 @@ def performSuccessfulBuildActions() { } } def String buildselector = "${BUILD_NUMBER}" - deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') - deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) - deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') - deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(meta, buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR', 'TOP_LEVEL_META_REPOSITORY') + deployjob(meta, buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') } def sendemails(emailinfo) { @@ -201,7 +205,7 @@ def sendextemail(extmailinfo) { ) } -def performFailedBuildActions() { +def performFailedBuildActions(meta) { def postbuildactions = null if (params.POST_BUILD_INFO != null) { def postbuildinfo = readPostBuildInfo() diff --git a/Common_scripts/differences/dometahierarchy.groovy b/Common_scripts/differences/dometahierarchy.groovy index 3b25798..f9a2404 100644 --- a/Common_scripts/differences/dometahierarchy.groovy +++ b/Common_scripts/differences/dometahierarchy.groovy @@ -27,7 +27,7 @@ class JiraTicketInfo { } class DifferenceRecord { - String productname + String productvariant String subcomponentname String rawdifferencetext @@ -39,7 +39,7 @@ class DifferenceRecord { String comment DifferenceRecord() { - productname = "" + productvariant = "" subcomponentname = "" rawdifferencetext = "" @@ -56,7 +56,7 @@ class DifferenceRecord { for (j in jiraticketinfo) { jiraticketstring += j.dump() + " " } - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + return "Difference Record:\nproductvariant: ${productvariant}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -130,7 +130,7 @@ def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { } def String getcomponent(meta) { - def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefolder = "${meta.productvariant()}_${meta.productinfo.subcomponentname}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" @@ -343,7 +343,7 @@ def List fillrecord(meta, line, String sourcepath) { def List differencerecords = [] def DifferenceRecord dr = new DifferenceRecord() - dr.productname = meta.productinfo.productname + dr.productvariant = meta.productvariant() dr.subcomponentname = meta.productinfo.subcomponentname dr.rawdifferencetext = line @@ -755,7 +755,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def title(earlymeta, latermeta) { return """ Differences in: - Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} + Product: ${earlymeta.productvariant()} Subcomponent: ${earlymeta.productinfo.subcomponentname} between: then: Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} @@ -771,7 +771,7 @@ def compare(earlymetaitems, latermetaitems, key) { def earlymeta = earlymetaitems[key].metainfo def latermeta = latermetaitems[key].metainfo - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: Changed: ") def boolean repoavailable def String uniquefolderforsourcerepo @@ -817,14 +817,14 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) differencerecords = differencerecords + componentdifferencerecords } else { - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) summaryappend("\n") } } for (key in latermetaitems.keySet()) { if (!earlymetaitems.containsKey(key)) { - summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + summaryappend("${latermeta.productvariant()} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) summaryappend("\n") } diff --git a/Common_scripts/documentation/doxygen_deploy.groovy b/Common_scripts/documentation/doxygen_deploy.groovy index 5054c02..e2f74a7 100644 --- a/Common_scripts/documentation/doxygen_deploy.groovy +++ b/Common_scripts/documentation/doxygen_deploy.groovy @@ -30,10 +30,10 @@ pipeline { importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) - productname = doxygenmeta.productinfo.productname + productvariant = doxygenmeta.productvariant() buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix - echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}" + echo "Doyxgen imported for product: ${productvariant} and build stamp prefix: ${buildstampprefix}" } } } @@ -41,7 +41,7 @@ pipeline { steps { script { doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" - pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}" + pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productvariant}/${buildstampprefix}" echo "Copying from ${doxygensource} to ${pathonserver}" if (isUnix()) { sh """#!/bin/bash diff --git a/Common_scripts/meta/folder.groovy b/Common_scripts/meta/folder.groovy index ac3d0dd..34448dd 100644 --- a/Common_scripts/meta/folder.groovy +++ b/Common_scripts/meta/folder.groovy @@ -817,20 +817,20 @@ class MetaInfo extends GISJsonSerialised { String relativize(String mypath) { String pathRelative - script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + //script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") if (mypath.startsWith(script.WORKSPACE)) { pathRelative = mypath.minus(script.WORKSPACE) - script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + //script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") } else { pathRelative = mypath - script.echo("Path relative is unchanged: ${pathRelative}") + //script.echo("Path relative is unchanged: ${pathRelative}") } // Convert \ with / pathRelative = pathRelative.tokenize('\\').join('/') // Strip leading / if any pathRelative = pathRelative.replaceAll('^/+', '') - script.echo("Path relative is ${pathRelative}") + //script.echo("Path relative is ${pathRelative}") return pathRelative } @@ -845,7 +845,7 @@ class MetaInfo extends GISJsonSerialised { def String adjustedpath = relativize(destpath) def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { - script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") + //script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) @@ -1103,7 +1103,7 @@ class MetaInfo extends GISJsonSerialised { def ArrayList metacollection = new ArrayList() def Map metamap = new LinkedHashMap() def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) - script.echo("Release walker iteration info at start:\n${ii.show()}") + //script.echo("Release walker iteration info at start:\n${ii.show()}") def results = walkrelated(ii) return results.metacollection } diff --git a/Common_scripts/product/info.groovy b/Common_scripts/product/info.groovy index 3d7bc30..67d1449 100644 --- a/Common_scripts/product/info.groovy +++ b/Common_scripts/product/info.groovy @@ -28,7 +28,6 @@ class ProductDependency { def setreleasestate() { def LinkedHashMap results = rtscript.startswithspecialprefix(presplitreleasetag) - script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix releasestate = results.prefixfound @@ -94,6 +93,10 @@ class ProductParsed { scanstate = State.PARSED } + def markparseerror() { + scanstate = State.PARSED + } + def boolean parsed() { return (scanstate == State.PARSED) } @@ -103,20 +106,43 @@ class ProductParsed { } } -def addProductEntry(LinkedHashMap products, +def String translateproductvariant(productvarianttranslation, + String releaserepo, + String productvariant) { + def translatedproductvariant = productvarianttranslation.get(releaserepo) + if (translatedproductvariant != null) { + echo "Translating product variant: ${productvariant} to: ${translatedproductvariant}" + return translatedproductvariant + } + return productvariant +} + +def ProductParsed addProductEntry(LinkedHashMap productcollection, + productvarianttranslation, String productvariant, String releaserepo) { - def productparsed - if (products.get(releaserepo) == null) { // Use the first entry name only - productparsed = new ProductParsed(productvariant) - products.put(releaserepo, productparsed) + def ProductParsed productparsed + + if (productvariant && releaserepo) { + if (productcollection.get(releaserepo) == null) { // Use the first entry name only + echo "Adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" + def translatedproductvariant = translateproductvariant(productvarianttranslation, + releaserepo, + productvariant) + productparsed = new ProductParsed(translatedproductvariant) + productcollection.put(releaserepo, productparsed) + } else { + echo "Already added product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" + } + productparsed = productcollection.get(releaserepo) } else { - productparsed = products.get(releaserepo) + echo "Not adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" } return productparsed } -def parseproductdependencies(LinkedHashMap products, +def parseproductdependencies(LinkedHashMap productcollection, + productvarianttranslation, productreleasemeta, String productreleasefolder, String containingproductvariant, @@ -124,88 +150,148 @@ def parseproductdependencies(LinkedHashMap products, def dependentmeta = productreleasemeta.getassociated(productreleasefolder) for (meta in dependentmeta) { - def pd = new ProductDependency(this, - rt, - meta.productvariant(), - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - pd.setreleasestate() - if (meta.productinfo.subcomponentname == 'WHOLE') { - if (meta.productvariant() != containingproductvariant) { // Don't add myself - addProductEntry(products, - meta.productvariant(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) + def translatedproductvariant = translateproductvariant(productvarianttranslation, + meta.releaseinfo.releaserepoURL, + meta.productvariant()) + if (translatedproductvariant && meta.productinfo.subcomponentname) { + def pd = new ProductDependency(this, + rt, + translatedproductvariant, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + pd.setreleasestate() + if (meta.productinfo.subcomponentname == 'WHOLE' && translatedproductvariant) { + if (translatedproductvariant != containingproductvariant) { // Don't add myself + addProductEntry(productcollection, + productvarianttranslation, + translatedproductvariant, + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } + } else { + def pc = new ProductComponent(meta.productinfo.subcomponentname, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductComponent(pc) } - } else if (meta.productvariant() != containingproductvariant) { - addProductEntry(products, - meta.productvariant(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) } else { - def pc = new ProductComponent(meta.productinfo.subcomponentname, - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductComponent(pc) + echo "Ignoring unnamed dependency in ${productreleasemeta.productvariant()}" } } } -def parseproductrepo(LinkedHashMap products, +def parseproductrepo(LinkedHashMap productcollection, + productvarianttranslation, String productrepo, String productreleasefolder) { - def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) - def String productvariant - def ProductParsed pp + def boolean productrepogood = true + echo "Querying repo: ${productrepo}" + dir(productreleasefolder) { + deleteDir() + def returnstatus = sh(script: """#!/bin/bash + git init + if [ \$? -ne 0 ]; then exit 1; fi + git config remote.origin.url ${productrepo} + if [ \$? -ne 0 ]; then exit 1; fi + git config advice.detachedHead false + if [ \$? -ne 0 ]; then exit 1; fi + git sparse-checkout init + if [ \$? -ne 0 ]; then exit 1; fi + git sparse-checkout set meta/* + if [ \$? -ne 0 ]; then exit 1; fi + git pull -t + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true) + if (returnstatus > 0) { + dir('meta') { + deleteDir() + } + echo "Couldn't clone repository: ${productrepo} correctly" + productrepogood = false + } + } + if (productrepogood) { + def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) + def String productvariant + def ProductParsed pp + shatagmap.each { sha, tags -> + //echo "Sha: ${sha} has tags: ${tags}" + dir(productreleasefolder) { + def returnstatus = sh(script: """#!/bin/bash + git checkout ${sha} + """, + returnStatus: true) + if (returnstatus > 0) { + dir('meta') { + deleteDir() + } + } + } + def productreleasemeta = meta.importrelated(false, productreleasefolder) + if (!productvariant) { + productvariant = productreleasemeta.productvariant() + pp = addProductEntry(productcollection, + productvarianttranslation, + productvariant, + productrepo) + if (pp) { + echo "Product variant created for: ${productrepo} name: ${pp.productvariant}" + } else { + echo "No product entry added/associated with ${productrepo} name: ${productvariant}" + } + } - def ArrayList productreleases = new ArrayList() - shatagmap.each { sha, tags -> - def importoptions = importer.getNewImportOptions() - importoptions.releaserepo(productrepo, sha) - importoptions.setfilter('meta/**') - def productreleasemeta = importer.artifactimport(meta, - importoptions, - productreleasefolder) - - if (!productvariant) { - productvariant = productreleasemeta.productvariant() - pp = addProductEntry(products, - productvariant, productrepo) + if (pp) { + def productrelease = new ProductRelease(sha, + productreleasemeta.jenkinsbuildinfo.builddate, + tags) + echo "Productrelease: ${productrelease.sha} created" + parseproductdependencies(productcollection, + productvarianttranslation, + productreleasemeta, + productreleasefolder, + pp.productvariant, + productrelease) + echo "Productrelease: ${productrelease.sha} added to Productvariant: ${pp.productvariant}" + pp.addProductRelease(productrelease) + } + } + if (pp) { + pp.markparsed() + echo "Productvariant: ${pp.productvariant} parsed" } - - def productrelease = new ProductRelease(sha, - productreleasemeta.jenkinsbuildinfo.builddate, - tags) - parseproductdependencies(products, - productreleasemeta, - productreleasefolder, - productvariant, - productrelease) - pp.addProductRelease(productrelease) } - pp.markparsed() } def iterateinitialproductrepos(initialproductcollection, + productvarianttranslation, LinkedHashMap productcollection, String productreleasefolder) { for (productrepo in initialproductcollection.ProductReleaseRepos) { - echo "Querying ${productrepo}" parseproductrepo(productcollection, + productvarianttranslation, productrepo, productreleasefolder) } while( true ) { boolean allparsed = true + echo "Parsing extra products" productcollection.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false parseproductrepo(productcollection, + productvarianttranslation, releaserepo, productreleasefolder) + } else { + echo "Parsed already ${productparsed.productvariant}" } } - if (allparsed) break + if (allparsed) { + echo "All parsed" + break + } } } @@ -213,5 +299,4 @@ def LinkedHashMap getNewProductCollection() { return new LinkedHashMap() } - return this \ No newline at end of file diff --git a/Common_scripts/publish/do.groovy b/Common_scripts/publish/do.groovy index ea090f0..8e302d8 100644 --- a/Common_scripts/publish/do.groovy +++ b/Common_scripts/publish/do.groovy @@ -22,7 +22,7 @@ def resetfolders() { } def productuniquefolder(releasemeta) { - return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + return "${releasemeta.productvariant()}_${releasemeta.productinfo.subcomponentname}" } def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { @@ -116,7 +116,7 @@ def recallcomponent(String productline, releaseinfo, String softwarerelease, ite def releasemeta = releaseinfo.metainfo def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + summaryappend("${releasemeta.productvariant()} ${releasemeta.productinfo.subcomponentname}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) @@ -145,7 +145,7 @@ def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, Str resetfolders() preconditions() - def productline = releasemeta.productinfo.productname + def productline = releasemeta.productvariant() // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) diff --git a/Common_scripts/publish/package.groovy b/Common_scripts/publish/package.groovy index 68c41f3..04236e0 100644 --- a/Common_scripts/publish/package.groovy +++ b/Common_scripts/publish/package.groovy @@ -8,30 +8,6 @@ pipeline { } } } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHONBRANCH = "origin/3.6.3_linux" - } - else { - PYTHONBRANCH = "origin/3.6.3_windows" - } - PYTHONDIR = "${WORKSPACE}/python" - PYTHONROOTDIR = 'python' - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) - if (isUnix()) { - env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" - } - else { - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - } - } - } stage('Clean exports') { steps { script { @@ -50,8 +26,16 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def packageimportoptions = importer.getNewImportOptions() - packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) - packageimportoptions.setincrementversionnumber(true) + // For backward compatibility - in case these parameters are not supplied + def packagelocation = params['PACKAGE_LOCATION'] ?: 'Jenkins job artifact' + def packagereleaserepo = params['PACKAGE_RELEASE_REPO'] ?: '' + def packagereleaserevision = params['PACKAGE_RELEASE_REVISION'] ?: '' + packageimportoptions.source(packagelocation, + PACKAGE_CREATE_JOB, + PACKAGE_CREATE_JOB_SELECTOR, + packagereleaserepo, + packagereleaserevision) + packageimportoptions.setincrementversionnumber(false) packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) PACKAGE_FOLDER = "${PACKAGEDEST}/package" diff --git a/Common_scripts/publish/packagecollection.groovy b/Common_scripts/publish/packagecollection.groovy new file mode 100644 index 0000000..a87453e --- /dev/null +++ b/Common_scripts/publish/packagecollection.groovy @@ -0,0 +1,183 @@ +def packagecollection = null + +/* Example expected JSON +{ +{ + "packages": [ + { + "Name": "gis-hmb-fd-hv-esw-el-sg1024", + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_app_0.9.6_debpackage", + "Job selector": "1248", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_package.git", + "Release repo revision": "refs/tag/0.9.6.57669", + "Path": "package/gis-hmb-fd-hv-esw-el-sg1024_armhf.deb" + }, + { + "Name": "gis-hmb-fd-hv-control-service-sg1024", + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_control_0.9.6_debpackage", + "Job selector": "1722", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_control_package.git", + "Release repo revision": "refs/tag/0.9.6.7964", + "Path": "package/gis-hmb-fd-hv-control-service-sg1024_armhf.deb" + } + ] +} +*/ + +def String asJsonText(somejson = packagecollection) { + return writeJSON(returnText: true, json: somejson, pretty: 1) +} + +def boolean packagecollection_exists() { + return (packagecollection != null) +} + +def packagecollection_set(String jsontext) { + packagecollection = readJSON(text: jsontext) + echo "Package collection info:\n${asJsonText(packagecollection)}\nType:\n${packagecollection.getClass()}" +} + +def packagecollection_get() { + return packagecollection +} + +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + PACKAGEDEST = "${WORKSPACE}/package" + cleanfolders = [ + PACKAGEDEST, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Transfer packages') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + packagecollection_set(PACKAGE_COLLECTION) + debpackagecollection = packagecollection_get() + + debpackagecollection.packages.each { debpackageinfo-> + echo "Processing package: ${debpackageinfo.Name}\n${debpackageinfo}\n${debpackageinfo.getClass()}" + echo "Info: ${debpackageinfo.Location}" + echo "Info: ${debpackageinfo['Location']}" + echo "Info: ${debpackageinfo['Job']}" + echo "Info: ${debpackageinfo['Job selector']}" + echo "Info: ${debpackageinfo['Release repo']}" + echo "Info: ${debpackageinfo['Release repo revision']}" + + def packagedest = "${PACKAGEDEST}/${debpackageinfo.Name}" + + def packageimportoptions = importer.getNewImportOptions() + // For backward compatibility - in case these parameters are not supplied + packageimportoptions.source(debpackageinfo['Location'], + debpackageinfo['Job'], + "${debpackageinfo['Job selector']}", + debpackageinfo['Release repo'], + debpackageinfo['Release repo revision']) + packageimportoptions.setincrementversionnumber(false) + packagemeta = importer.artifactimport(meta, packageimportoptions, packagedest) + + UNIQUE_PACKAGE_FOLDER = "${packagedest}/package" + + echo "Aptly config file is ${APTLY_CONFIG_FILE}" + echo "Repo name is ${REPO_NAME}" + echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" + + sh """#!/bin/bash -v + aptly config show + if [ \$? -ne 0 ]; then exit 1; fi + + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then + echo "aptly repo ${REPO_NAME} does not exist; Creating it" + aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + fi + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" + // Convert prefix to regular expression + PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") + PACKAGE_VERSION_REGEX = '' + i = 0 + while(true) { + if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + break; + } + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + PACKAGE_VERSION_REGEX += '\\.' + i++ + } + sh """#!/bin/bash -v + # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' + if [ \$? -ne 0 ]; then exit 1; fi + aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/Common_scripts/release/do.groovy b/Common_scripts/release/do.groovy index 4dcb091..05de91f 100644 --- a/Common_scripts/release/do.groovy +++ b/Common_scripts/release/do.groovy @@ -191,7 +191,7 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}" taglist.add(componentothertagitem) } } @@ -377,13 +377,13 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { def buildmeta = buildinfo.metainfo def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) - def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" + def String uniquefolder = "${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") - repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + summaryappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) diff --git a/Common_scripts/somsoftware/sominfo.groovy b/Common_scripts/somsoftware/sominfo.groovy new file mode 100644 index 0000000..a7fdc82 --- /dev/null +++ b/Common_scripts/somsoftware/sominfo.groovy @@ -0,0 +1,31 @@ +def somsoftwareinfostore = null + +/* Example expected JSON +{ + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "SOM_GIS_master/SOM_GIS_master_build", + "Job selector": "true", + "Release repo": "git@hydra:som_gis-releases/som_gis_whole.git", + "Release repo revision": "" + } +}*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: somsoftwareinfostore, pretty: 1) +} + +def boolean exists() { + return (somsoftwareinfostore != null) +} + +def set(String jsontext) { + somsoftwareinfostore = readJSON(text: jsontext) + echo "Som software support info:\n${asJsonText()}" +} + +def get() { + return somsoftwareinfostore +} + +return this \ No newline at end of file diff --git a/Common_scripts/versioncontrol/git.groovy b/Common_scripts/versioncontrol/git.groovy index 61378b5..2b6fb09 100644 --- a/Common_scripts/versioncontrol/git.groovy +++ b/Common_scripts/versioncontrol/git.groovy @@ -43,7 +43,6 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL def shatagfields = shatag.split('\\t') def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() - echo "sha: ${sha} tag: ${tag}" if (shatagmap.get(sha) == null) { def ArrayList taglist = new ArrayList() taglist.add(tag) @@ -54,6 +53,9 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL shatagmap.put(sha, taglist) } } + shatagmap.each { sha, tags -> + echo "sha: ${sha} has tags: ${tags}" + } return shatagmap }