From ef15ab68681df6a42674955d4362dcb2444bb664 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 23 Mar 2022 08:19:15 +0000 Subject: [PATCH] Squashed 'Common_scripts/' changes from b70fe5f..791c94c 791c94c Pass meta to post action handlers; Pass release repo to post actions d21f4b1 Pass meta to post action handlers; Pass release repo to post actions 5e6c640 Merge remote-tracking branch 'origin/master' into production 7c05dde Use productvariant() instead of .productinfo.productname 8cd7ae9 Merge commit '0bb8e883772eb23ba839d6b934857b78fc1f5317' into production 2b61646 Products are those with WHOLE subcomponent 83257ea Merge commit 'a29f25f7be570e0c07a12f35055f8818cef47fa7' into production 062711c Merge commit 'ac6ed3de46417d119c9bd12ca1b1879976f99248' 2ae44ff Use productvariant() instead of .productinfo.productname 2e74b32 Provide a product variant translation table 806f098 Provide a product variant translation table 5db7c8d Provide a product variant translation table 3e317a5 Provide a product variant translation table 58d6990 Provide a product variant translation table ba8062d Provide a product variant translation table 14b1eff Provide a product variant translation table bf4a428 Debug sha/tags table e72f006 Debug sha/tags table 295c18c Debug sha/tags table 8aeb9a9 Debug sha/tags table ab4165e Debug sha/tags table 839f987 Debug sha/tags table cd0a353 Debug sha/tags table 94de9d2 Debug sha/tags table 60c1fba Debug sha/tags table 390c97c Debug sha/tags table da33b16 Debug sha/tags table 9235c1a Debug sha/tags table eb960a6 Debug sha/tags table 9f0a5db Debug sha/tags table 93b2f2b Debug sha/tags table e711a6c Debug sha/tags table 149aad2 Debug sha/tags table fd795d2 Debug sha/tags table 2580ed3 Debug sha/tags table 66a6672 Debug sha/tags table df5c9d1 Remove debug 82f9bb6 Use raw git to checkout each tag 1c9579f Use raw git to checkout each tag f55ca99 Merge commit '610e4be77e08d4b03b8157e38c099c43b2d6834e' into production 7682029 Merge commit 'e8e55db451e17f7b3edaab583e77a0428cebdfe7' into master 1810f08 Merge commit '52965d83743e36da81c13d1b0e98e29244fbc095' 2f496a5 Merge commit '7707b2cffa0e4c308bb6453f5ef3b0eeffc3acf8' 4ea46e3 Merge commit 'dea77dfa289efef365b5d1b48262c089832a5fa3' 85afca9 Create a script for publishing a set of debian packages to a remote update server e408bd1 Create a script for publishing a set of debian packages to a remote update server 6fd010a Create a script for publishing a set of debian packages to a remote update server 7515c39 Create a script for publishing a set of debian packages to a remote update server 0c5fb98 Create a script for publishing a set of debian packages to a remote update server 9ad83e9 Create a script for publishing a set of debian packages to a remote update server 80c2a61 Create a script for publishing a set of debian packages to a remote update server b0cc38c Create a script for publishing a set of debian packages to a remote update server 2c11a0c Create a script for publishing a set of debian packages to a remote update server 7e987f0 Create a script for publishing a set of debian packages to a remote update server f9c8438 Create a script for publishing a set of debian packages to a remote update server 46fc969 Create a script for publishing a set of debian packages to a remote update server b72ddc8 Create a script for publishing a set of debian packages to a remote update server 1beb23e Create a script for publishing a set of debian packages to a remote update server 97b31de Create a script for publishing a set of debian packages to a remote update server 9285f75 Create a script for publishing a set of debian packages to a remote update server d4513a8 Merge remote-tracking branch 'origin/production' b0d92b0 CORE-348 - Provide a script that can obtain a package from a release repo aa124fa CORE-348 - Provide a script that can obtain a package from a release repo 05f9c1b CORE-348 - Provide a script that can obtain a package from a release repo ddd07b9 CORE-348 - Provide a script that can obtain a package from a release repo ae97402 CORE-348 - Build a som-only dependent FDHV package git-subtree-dir: Common_scripts git-subtree-split: 791c94c94d28ee29daf58b55738cd9c13cd158eb --- build/postactions.groovy | 30 ++-- differences/dometahierarchy.groovy | 18 +-- documentation/doxygen_deploy.groovy | 6 +- meta/folder.groovy | 12 +- product/info.groovy | 205 ++++++++++++++++++++-------- publish/do.groovy | 6 +- publish/package.groovy | 36 ++--- publish/packagecollection.groovy | 183 +++++++++++++++++++++++++ release/do.groovy | 8 +- somsoftware/sominfo.groovy | 31 +++++ versioncontrol/git.groovy | 4 +- 11 files changed, 414 insertions(+), 125 deletions(-) create mode 100644 publish/packagecollection.groovy create mode 100644 somsoftware/sominfo.groovy diff --git a/build/postactions.groovy b/build/postactions.groovy index 8b18908..3206004 100644 --- a/build/postactions.groovy +++ b/build/postactions.groovy @@ -85,16 +85,20 @@ def getActionJobName(postbuildactions, String action, String alternativeparamnam return jobname } -def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { +def deployjob(meta, String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername, String releaserepoparametername = '') { def jobname = getActionJobName(postbuildactions, action, alternativeparamname) if (jobname) { + def parameters = [ + new StringParameterValue(jobnameparametername, JOB_NAME), + new StringParameterValue(jobselectorparametername, buildselector), + ] + if (releaserepoparametername) { + parameters += new StringParameterValue(releaserepoparametername, meta.releaseinfo.releaserepoURL) + } build( wait: false, job: jobname, - parameters: [ - string(name: jobnameparametername, value: JOB_NAME), - string(name: jobselectorparametername, value: buildselector), - ] + parameters: parameters ) } } @@ -107,7 +111,7 @@ def readPostBuildInfo() { return postbuildinfo } -def performSuccessfulBuildActions() { +def performSuccessfulBuildActions(meta) { def postbuildactions if (params.POST_BUILD_INFO != null) { def postbuildinfo = readPostBuildInfo() @@ -119,12 +123,12 @@ def performSuccessfulBuildActions() { } } def String buildselector = "${BUILD_NUMBER}" - deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') - deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) - deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') - deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') - deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(meta, buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(meta, buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR', 'TOP_LEVEL_META_REPOSITORY') + deployjob(meta, buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') } def sendemails(emailinfo) { @@ -201,7 +205,7 @@ def sendextemail(extmailinfo) { ) } -def performFailedBuildActions() { +def performFailedBuildActions(meta) { def postbuildactions = null if (params.POST_BUILD_INFO != null) { def postbuildinfo = readPostBuildInfo() diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 3b25798..f9a2404 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -27,7 +27,7 @@ class JiraTicketInfo { } class DifferenceRecord { - String productname + String productvariant String subcomponentname String rawdifferencetext @@ -39,7 +39,7 @@ class DifferenceRecord { String comment DifferenceRecord() { - productname = "" + productvariant = "" subcomponentname = "" rawdifferencetext = "" @@ -56,7 +56,7 @@ class DifferenceRecord { for (j in jiraticketinfo) { jiraticketstring += j.dump() + " " } - return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + return "Difference Record:\nproductvariant: ${productvariant}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } @@ -130,7 +130,7 @@ def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { } def String getcomponent(meta) { - def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefolder = "${meta.productvariant()}_${meta.productinfo.subcomponentname}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" @@ -343,7 +343,7 @@ def List fillrecord(meta, line, String sourcepath) { def List differencerecords = [] def DifferenceRecord dr = new DifferenceRecord() - dr.productname = meta.productinfo.productname + dr.productvariant = meta.productvariant() dr.subcomponentname = meta.productinfo.subcomponentname dr.rawdifferencetext = line @@ -755,7 +755,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def title(earlymeta, latermeta) { return """ Differences in: - Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} + Product: ${earlymeta.productvariant()} Subcomponent: ${earlymeta.productinfo.subcomponentname} between: then: Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} @@ -771,7 +771,7 @@ def compare(earlymetaitems, latermetaitems, key) { def earlymeta = earlymetaitems[key].metainfo def latermeta = latermetaitems[key].metainfo - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: Changed: ") def boolean repoavailable def String uniquefolderforsourcerepo @@ -817,14 +817,14 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) differencerecords = differencerecords + componentdifferencerecords } else { - summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + summaryappend("${earlymeta.productvariant()} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) summaryappend("\n") } } for (key in latermetaitems.keySet()) { if (!earlymetaitems.containsKey(key)) { - summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + summaryappend("${latermeta.productvariant()} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) summaryappend("\n") } diff --git a/documentation/doxygen_deploy.groovy b/documentation/doxygen_deploy.groovy index 5054c02..e2f74a7 100644 --- a/documentation/doxygen_deploy.groovy +++ b/documentation/doxygen_deploy.groovy @@ -30,10 +30,10 @@ pipeline { importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**') doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN) - productname = doxygenmeta.productinfo.productname + productvariant = doxygenmeta.productvariant() buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix - echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}" + echo "Doyxgen imported for product: ${productvariant} and build stamp prefix: ${buildstampprefix}" } } } @@ -41,7 +41,7 @@ pipeline { steps { script { doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}" - pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}" + pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productvariant}/${buildstampprefix}" echo "Copying from ${doxygensource} to ${pathonserver}" if (isUnix()) { sh """#!/bin/bash diff --git a/meta/folder.groovy b/meta/folder.groovy index ac3d0dd..34448dd 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -817,20 +817,20 @@ class MetaInfo extends GISJsonSerialised { String relativize(String mypath) { String pathRelative - script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + //script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") if (mypath.startsWith(script.WORKSPACE)) { pathRelative = mypath.minus(script.WORKSPACE) - script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + //script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") } else { pathRelative = mypath - script.echo("Path relative is unchanged: ${pathRelative}") + //script.echo("Path relative is unchanged: ${pathRelative}") } // Convert \ with / pathRelative = pathRelative.tokenize('\\').join('/') // Strip leading / if any pathRelative = pathRelative.replaceAll('^/+', '') - script.echo("Path relative is ${pathRelative}") + //script.echo("Path relative is ${pathRelative}") return pathRelative } @@ -845,7 +845,7 @@ class MetaInfo extends GISJsonSerialised { def String adjustedpath = relativize(destpath) def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { - script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") + //script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) @@ -1103,7 +1103,7 @@ class MetaInfo extends GISJsonSerialised { def ArrayList metacollection = new ArrayList() def Map metamap = new LinkedHashMap() def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) - script.echo("Release walker iteration info at start:\n${ii.show()}") + //script.echo("Release walker iteration info at start:\n${ii.show()}") def results = walkrelated(ii) return results.metacollection } diff --git a/product/info.groovy b/product/info.groovy index 3d7bc30..67d1449 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -28,7 +28,6 @@ class ProductDependency { def setreleasestate() { def LinkedHashMap results = rtscript.startswithspecialprefix(presplitreleasetag) - script.echo("results is: ${results.dump()}") if (results.hasspecialprefix) { releasetag = results.tagwithoutprefix releasestate = results.prefixfound @@ -94,6 +93,10 @@ class ProductParsed { scanstate = State.PARSED } + def markparseerror() { + scanstate = State.PARSED + } + def boolean parsed() { return (scanstate == State.PARSED) } @@ -103,20 +106,43 @@ class ProductParsed { } } -def addProductEntry(LinkedHashMap products, +def String translateproductvariant(productvarianttranslation, + String releaserepo, + String productvariant) { + def translatedproductvariant = productvarianttranslation.get(releaserepo) + if (translatedproductvariant != null) { + echo "Translating product variant: ${productvariant} to: ${translatedproductvariant}" + return translatedproductvariant + } + return productvariant +} + +def ProductParsed addProductEntry(LinkedHashMap productcollection, + productvarianttranslation, String productvariant, String releaserepo) { - def productparsed - if (products.get(releaserepo) == null) { // Use the first entry name only - productparsed = new ProductParsed(productvariant) - products.put(releaserepo, productparsed) + def ProductParsed productparsed + + if (productvariant && releaserepo) { + if (productcollection.get(releaserepo) == null) { // Use the first entry name only + echo "Adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" + def translatedproductvariant = translateproductvariant(productvarianttranslation, + releaserepo, + productvariant) + productparsed = new ProductParsed(translatedproductvariant) + productcollection.put(releaserepo, productparsed) + } else { + echo "Already added product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" + } + productparsed = productcollection.get(releaserepo) } else { - productparsed = products.get(releaserepo) + echo "Not adding product entry: releaserepo: ${releaserepo} (product variant: ${productvariant})" } return productparsed } -def parseproductdependencies(LinkedHashMap products, +def parseproductdependencies(LinkedHashMap productcollection, + productvarianttranslation, productreleasemeta, String productreleasefolder, String containingproductvariant, @@ -124,88 +150,148 @@ def parseproductdependencies(LinkedHashMap products, def dependentmeta = productreleasemeta.getassociated(productreleasefolder) for (meta in dependentmeta) { - def pd = new ProductDependency(this, - rt, - meta.productvariant(), - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - pd.setreleasestate() - if (meta.productinfo.subcomponentname == 'WHOLE') { - if (meta.productvariant() != containingproductvariant) { // Don't add myself - addProductEntry(products, - meta.productvariant(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) + def translatedproductvariant = translateproductvariant(productvarianttranslation, + meta.releaseinfo.releaserepoURL, + meta.productvariant()) + if (translatedproductvariant && meta.productinfo.subcomponentname) { + def pd = new ProductDependency(this, + rt, + translatedproductvariant, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + pd.setreleasestate() + if (meta.productinfo.subcomponentname == 'WHOLE' && translatedproductvariant) { + if (translatedproductvariant != containingproductvariant) { // Don't add myself + addProductEntry(productcollection, + productvarianttranslation, + translatedproductvariant, + meta.releaseinfo.releaserepoURL) + productrelease.addProductDependency(pd) + } + } else { + def pc = new ProductComponent(meta.productinfo.subcomponentname, + meta.releasetag(), + meta.releaseinfo.releaserepoURL) + productrelease.addProductComponent(pc) } - } else if (meta.productvariant() != containingproductvariant) { - addProductEntry(products, - meta.productvariant(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductDependency(pd) } else { - def pc = new ProductComponent(meta.productinfo.subcomponentname, - meta.releasetag(), - meta.releaseinfo.releaserepoURL) - productrelease.addProductComponent(pc) + echo "Ignoring unnamed dependency in ${productreleasemeta.productvariant()}" } } } -def parseproductrepo(LinkedHashMap products, +def parseproductrepo(LinkedHashMap productcollection, + productvarianttranslation, String productrepo, String productreleasefolder) { - def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) - def String productvariant - def ProductParsed pp + def boolean productrepogood = true + echo "Querying repo: ${productrepo}" + dir(productreleasefolder) { + deleteDir() + def returnstatus = sh(script: """#!/bin/bash + git init + if [ \$? -ne 0 ]; then exit 1; fi + git config remote.origin.url ${productrepo} + if [ \$? -ne 0 ]; then exit 1; fi + git config advice.detachedHead false + if [ \$? -ne 0 ]; then exit 1; fi + git sparse-checkout init + if [ \$? -ne 0 ]; then exit 1; fi + git sparse-checkout set meta/* + if [ \$? -ne 0 ]; then exit 1; fi + git pull -t + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true) + if (returnstatus > 0) { + dir('meta') { + deleteDir() + } + echo "Couldn't clone repository: ${productrepo} correctly" + productrepogood = false + } + } + if (productrepogood) { + def LinkedHashMap> shatagmap = gh.getremoteshatagmap(productrepo) + def String productvariant + def ProductParsed pp + shatagmap.each { sha, tags -> + //echo "Sha: ${sha} has tags: ${tags}" + dir(productreleasefolder) { + def returnstatus = sh(script: """#!/bin/bash + git checkout ${sha} + """, + returnStatus: true) + if (returnstatus > 0) { + dir('meta') { + deleteDir() + } + } + } + def productreleasemeta = meta.importrelated(false, productreleasefolder) + if (!productvariant) { + productvariant = productreleasemeta.productvariant() + pp = addProductEntry(productcollection, + productvarianttranslation, + productvariant, + productrepo) + if (pp) { + echo "Product variant created for: ${productrepo} name: ${pp.productvariant}" + } else { + echo "No product entry added/associated with ${productrepo} name: ${productvariant}" + } + } - def ArrayList productreleases = new ArrayList() - shatagmap.each { sha, tags -> - def importoptions = importer.getNewImportOptions() - importoptions.releaserepo(productrepo, sha) - importoptions.setfilter('meta/**') - def productreleasemeta = importer.artifactimport(meta, - importoptions, - productreleasefolder) - - if (!productvariant) { - productvariant = productreleasemeta.productvariant() - pp = addProductEntry(products, - productvariant, productrepo) + if (pp) { + def productrelease = new ProductRelease(sha, + productreleasemeta.jenkinsbuildinfo.builddate, + tags) + echo "Productrelease: ${productrelease.sha} created" + parseproductdependencies(productcollection, + productvarianttranslation, + productreleasemeta, + productreleasefolder, + pp.productvariant, + productrelease) + echo "Productrelease: ${productrelease.sha} added to Productvariant: ${pp.productvariant}" + pp.addProductRelease(productrelease) + } + } + if (pp) { + pp.markparsed() + echo "Productvariant: ${pp.productvariant} parsed" } - - def productrelease = new ProductRelease(sha, - productreleasemeta.jenkinsbuildinfo.builddate, - tags) - parseproductdependencies(products, - productreleasemeta, - productreleasefolder, - productvariant, - productrelease) - pp.addProductRelease(productrelease) } - pp.markparsed() } def iterateinitialproductrepos(initialproductcollection, + productvarianttranslation, LinkedHashMap productcollection, String productreleasefolder) { for (productrepo in initialproductcollection.ProductReleaseRepos) { - echo "Querying ${productrepo}" parseproductrepo(productcollection, + productvarianttranslation, productrepo, productreleasefolder) } while( true ) { boolean allparsed = true + echo "Parsing extra products" productcollection.each { releaserepo, productparsed -> if (!productparsed.parsed()) { allparsed = false parseproductrepo(productcollection, + productvarianttranslation, releaserepo, productreleasefolder) + } else { + echo "Parsed already ${productparsed.productvariant}" } } - if (allparsed) break + if (allparsed) { + echo "All parsed" + break + } } } @@ -213,5 +299,4 @@ def LinkedHashMap getNewProductCollection() { return new LinkedHashMap() } - return this \ No newline at end of file diff --git a/publish/do.groovy b/publish/do.groovy index ea090f0..8e302d8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -22,7 +22,7 @@ def resetfolders() { } def productuniquefolder(releasemeta) { - return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + return "${releasemeta.productvariant()}_${releasemeta.productinfo.subcomponentname}" } def uniquereleasefolder(String productline, String softwarerelease, releasemeta) { @@ -116,7 +116,7 @@ def recallcomponent(String productline, releaseinfo, String softwarerelease, ite def releasemeta = releaseinfo.metainfo def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) - summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + summaryappend("${releasemeta.productvariant()} ${releasemeta.productinfo.subcomponentname}: ") getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) @@ -145,7 +145,7 @@ def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, Str resetfolders() preconditions() - def productline = releasemeta.productinfo.productname + def productline = releasemeta.productvariant() // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) diff --git a/publish/package.groovy b/publish/package.groovy index 68c41f3..04236e0 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -8,30 +8,6 @@ pipeline { } } } - stage('Checkout dependencies') { - steps { - script { - configurationhelper = autobuilder.configurationhelper() - - PYTHONURL = 'git@hydra:tools/python.git' - if (isUnix()) { - PYTHONBRANCH = "origin/3.6.3_linux" - } - else { - PYTHONBRANCH = "origin/3.6.3_windows" - } - PYTHONDIR = "${WORKSPACE}/python" - PYTHONROOTDIR = 'python' - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) - if (isUnix()) { - env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" - } - else { - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - } - } - } - } stage('Clean exports') { steps { script { @@ -50,8 +26,16 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def packageimportoptions = importer.getNewImportOptions() - packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) - packageimportoptions.setincrementversionnumber(true) + // For backward compatibility - in case these parameters are not supplied + def packagelocation = params['PACKAGE_LOCATION'] ?: 'Jenkins job artifact' + def packagereleaserepo = params['PACKAGE_RELEASE_REPO'] ?: '' + def packagereleaserevision = params['PACKAGE_RELEASE_REVISION'] ?: '' + packageimportoptions.source(packagelocation, + PACKAGE_CREATE_JOB, + PACKAGE_CREATE_JOB_SELECTOR, + packagereleaserepo, + packagereleaserevision) + packageimportoptions.setincrementversionnumber(false) packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) PACKAGE_FOLDER = "${PACKAGEDEST}/package" diff --git a/publish/packagecollection.groovy b/publish/packagecollection.groovy new file mode 100644 index 0000000..a87453e --- /dev/null +++ b/publish/packagecollection.groovy @@ -0,0 +1,183 @@ +def packagecollection = null + +/* Example expected JSON +{ +{ + "packages": [ + { + "Name": "gis-hmb-fd-hv-esw-el-sg1024", + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_app_0.9.6_debpackage", + "Job selector": "1248", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_package.git", + "Release repo revision": "refs/tag/0.9.6.57669", + "Path": "package/gis-hmb-fd-hv-esw-el-sg1024_armhf.deb" + }, + { + "Name": "gis-hmb-fd-hv-control-service-sg1024", + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_control_0.9.6_debpackage", + "Job selector": "1722", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_control_package.git", + "Release repo revision": "refs/tag/0.9.6.7964", + "Path": "package/gis-hmb-fd-hv-control-service-sg1024_armhf.deb" + } + ] +} +*/ + +def String asJsonText(somejson = packagecollection) { + return writeJSON(returnText: true, json: somejson, pretty: 1) +} + +def boolean packagecollection_exists() { + return (packagecollection != null) +} + +def packagecollection_set(String jsontext) { + packagecollection = readJSON(text: jsontext) + echo "Package collection info:\n${asJsonText(packagecollection)}\nType:\n${packagecollection.getClass()}" +} + +def packagecollection_get() { + return packagecollection +} + +pipeline { + agent { label AGENT_LABEL } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + PACKAGEDEST = "${WORKSPACE}/package" + cleanfolders = [ + PACKAGEDEST, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Transfer packages') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + packagecollection_set(PACKAGE_COLLECTION) + debpackagecollection = packagecollection_get() + + debpackagecollection.packages.each { debpackageinfo-> + echo "Processing package: ${debpackageinfo.Name}\n${debpackageinfo}\n${debpackageinfo.getClass()}" + echo "Info: ${debpackageinfo.Location}" + echo "Info: ${debpackageinfo['Location']}" + echo "Info: ${debpackageinfo['Job']}" + echo "Info: ${debpackageinfo['Job selector']}" + echo "Info: ${debpackageinfo['Release repo']}" + echo "Info: ${debpackageinfo['Release repo revision']}" + + def packagedest = "${PACKAGEDEST}/${debpackageinfo.Name}" + + def packageimportoptions = importer.getNewImportOptions() + // For backward compatibility - in case these parameters are not supplied + packageimportoptions.source(debpackageinfo['Location'], + debpackageinfo['Job'], + "${debpackageinfo['Job selector']}", + debpackageinfo['Release repo'], + debpackageinfo['Release repo revision']) + packageimportoptions.setincrementversionnumber(false) + packagemeta = importer.artifactimport(meta, packageimportoptions, packagedest) + + UNIQUE_PACKAGE_FOLDER = "${packagedest}/package" + + echo "Aptly config file is ${APTLY_CONFIG_FILE}" + echo "Repo name is ${REPO_NAME}" + echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" + + sh """#!/bin/bash -v + aptly config show + if [ \$? -ne 0 ]; then exit 1; fi + + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then + echo "aptly repo ${REPO_NAME} does not exist; Creating it" + aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + fi + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" + // Convert prefix to regular expression + PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]") + PACKAGE_VERSION_REGEX = '' + i = 0 + while(true) { + if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) { + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + break; + } + PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i] + PACKAGE_VERSION_REGEX += '\\.' + i++ + } + sh """#!/bin/bash -v + # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)' + if [ \$? -ne 0 ]; then exit 1; fi + aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/do.groovy b/release/do.groovy index 4dcb091..05de91f 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -191,7 +191,7 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { if (othertagitem?.trim()) { - def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}" taglist.add(componentothertagitem) } } @@ -377,13 +377,13 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { def buildmeta = buildinfo.metainfo def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) - def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" + def String uniquefolder = "${buildmeta.productvariant()}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" - summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") - repoandtagsappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + summaryappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") + repoandtagsappend("${buildmeta.productvariant()} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) diff --git a/somsoftware/sominfo.groovy b/somsoftware/sominfo.groovy new file mode 100644 index 0000000..a7fdc82 --- /dev/null +++ b/somsoftware/sominfo.groovy @@ -0,0 +1,31 @@ +def somsoftwareinfostore = null + +/* Example expected JSON +{ + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "SOM_GIS_master/SOM_GIS_master_build", + "Job selector": "true", + "Release repo": "git@hydra:som_gis-releases/som_gis_whole.git", + "Release repo revision": "" + } +}*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: somsoftwareinfostore, pretty: 1) +} + +def boolean exists() { + return (somsoftwareinfostore != null) +} + +def set(String jsontext) { + somsoftwareinfostore = readJSON(text: jsontext) + echo "Som software support info:\n${asJsonText()}" +} + +def get() { + return somsoftwareinfostore +} + +return this \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 61378b5..2b6fb09 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -43,7 +43,6 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL def shatagfields = shatag.split('\\t') def sha = shatagfields[0].trim() def tag = shatagfields[1].trim() - echo "sha: ${sha} tag: ${tag}" if (shatagmap.get(sha) == null) { def ArrayList taglist = new ArrayList() taglist.add(tag) @@ -54,6 +53,9 @@ LinkedHashMap> getremoteshatagmap(String repositoryURL shatagmap.put(sha, taglist) } } + shatagmap.each { sha, tags -> + echo "sha: ${sha} has tags: ${tags}" + } return shatagmap }