From c08e33f026fe49cfe18cadfdc9aa18de65fb69a9 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Thu, 19 Aug 2021 16:07:32 +0100 Subject: [PATCH 01/15] ELGIP-139 remove output.txt before we start otherwise errors persist and the file keeps growing --- autobuild/synthesis/build_image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/autobuild/synthesis/build_image.sh b/autobuild/synthesis/build_image.sh index d49d39a..dc87d5f 100644 --- a/autobuild/synthesis/build_image.sh +++ b/autobuild/synthesis/build_image.sh @@ -34,6 +34,10 @@ fi # set QPF_TOP=top_system # set QSYS_TOP=top_system +if [ -f "$WORKSPACE/output.txt" ]; then + rm -f "$WORKSPACE/output.txt" +fi + # Check if Qsys exists if [ -f "$PROJECT_DIR/${QSYS_TOP}.qsys" ]; then From f40876abeebbd47e483264cee08295a2b15c1eb2 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Thu, 2 Sep 2021 13:59:55 +0100 Subject: [PATCH 02/15] ELGIP-139 check which type of versioning is needed --- autobuild/synthesis/synthesis.py | 39 +++++++++++++++++--------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/autobuild/synthesis/synthesis.py b/autobuild/synthesis/synthesis.py index b0bc653..fba24bb 100644 --- a/autobuild/synthesis/synthesis.py +++ b/autobuild/synthesis/synthesis.py @@ -7,28 +7,31 @@ from distutils.util import strtobool WORKSPACE=os.environ["WORKSPACE"] print("WORKSPACE is {0}".format(WORKSPACE)) -if (("PREVENT_AUTO_VERSIONING" in os.environ) and ('true' == os.environ["PREVENT_AUTO_VERSIONING"])) : - print("NOT performing AUTO Versioning") -else : - from update_fw_ver import updateFwVer - - if os.path.exists("el/firmware/el_generic_ip") : - versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/el_generic_ip/generic_ip/generic_versioning/doc/generic_versioning.json")) +if os.path.exists("el/firmware/common/scripts/create_pkg.sh"): + print("versioning for Non-QSYS products will be handled later in script") +else: + if (("PREVENT_AUTO_VERSIONING" in os.environ) and ('true' == os.environ["PREVENT_AUTO_VERSIONING"])) : + print("NOT performing AUTO Versioning") else : - versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/generic_ip/generic_versioning/doc/generic_versioning.json")) + from update_fw_ver import updateFwVer - try: - buildStamp=os.environ["BUILD_STAMP"] - except: - print("unable to access build_stamp env variable") - sys.exit(1) + if os.path.exists("el/firmware/el_generic_ip") : + versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/el_generic_ip/generic_ip/generic_versioning/doc/generic_versioning.json")) + else : + versioningJsonFileName=os.path.normpath(os.path.join(WORKSPACE,"el/firmware/generic_ip/generic_versioning/doc/generic_versioning.json")) - try: - if (0 != updateFwVer(versioningJsonFileName,buildStamp)) : + try: + buildStamp=os.environ["BUILD_STAMP"] + except: + print("unable to access build_stamp env variable") + sys.exit(1) + + try: + if (0 != updateFwVer(versioningJsonFileName,buildStamp)) : + sys.exit(1) + except: + print("Error updating fw version") sys.exit(1) - except: - print("Error updating fw version") - sys.exit(1) QPF_TOP=os.environ["QPF_TOP"] QSYS_TOP=os.environ["QSYS_TOP"] From 98c848b6ab1df124fdc0cf67e28f34520a2c1b6e Mon Sep 17 00:00:00 2001 From: Dave Steele Date: Thu, 23 Sep 2021 15:13:56 +0100 Subject: [PATCH 03/15] CORE-491. Embedded Logic Simulation library allows illegal commands to be built for the datastreamer. Print error message --- verification/lib/tests/utils/dataStreamer7Utils.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/verification/lib/tests/utils/dataStreamer7Utils.h b/verification/lib/tests/utils/dataStreamer7Utils.h index 67200d1..e3301a0 100644 --- a/verification/lib/tests/utils/dataStreamer7Utils.h +++ b/verification/lib/tests/utils/dataStreamer7Utils.h @@ -12,7 +12,7 @@ class dataStreamer7Utils { public: const uint32_t numCmdsInBlock = 15; - + // Bit offsets of data streamer commands enum DataStreamerCmdOffsets_e { dsCmdOffset = 0, @@ -32,7 +32,7 @@ public: loadCmdQPaddr = 0x4, nop = 0xf }; - + // Valid data streamer destinations. Note cmdQueueDest is only for load command qwueue commands enum headDest_e { headDest0 = 0, @@ -76,6 +76,8 @@ protected: ((dest & 0x0f) << dsDestOffset) | ((loop & 0xff) << dsLoopOffset) | ((param0 & 0xfff) << dsParam0Offset); + if (cmd == rdDataPaddr && (param0 & 0xfff) > 256) + VPrint("***** Error WriteDsCmd rdDataPaddr parameter 0 (numwords) %d > 256 max\n", param0 & 0xfff); } return word; }; From daffc3818bae89721e395dc3d74f7eb91e3daf5a Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Fri, 1 Oct 2021 11:01:42 +0100 Subject: [PATCH 04/15] ELGIP-172 - use genericsequencer16 to match real code --- verification/lib/tests/utils/fpm6Utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verification/lib/tests/utils/fpm6Utils.h b/verification/lib/tests/utils/fpm6Utils.h index 500d637..65fb9ce 100644 --- a/verification/lib/tests/utils/fpm6Utils.h +++ b/verification/lib/tests/utils/fpm6Utils.h @@ -99,7 +99,7 @@ protected: CSR_PULSE_GENERATOR_3_FRACTIONAL_DIVIDER_CONTROL_UDP_MIN_POS_MASK); // enable the ch-ctrl sequencer - pPulseGen->pGenericSequencer15Channel->pCtrl->SetSeqEnable(1); + pPulseGen->pGenericSequencer16Channel->pCtrl->SetSeqEnable(1); // Issue a start_req to the ch-ctrl sequencer From b6627125687988bd75d9de050cce23f92111a882 Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 5 Oct 2021 13:29:25 +0100 Subject: [PATCH 05/15] Correction for IP Checksum as per email from Simon 19/9/21 I found a bug in the TCP/IPV4 checksum method where an odd sized payload would falsely check as bad if the rx buffer happened to have a non zero value in the byte after the payload. Just in case this became an issue, I thought I would let you know the fix. In tcpClient.cpp, in the ipv4_checksum method the 'for' loop is changed to: for (int idx = 0; idx < (len-1); idx+=2) And that's it. best regards, Simon --- verification/lib/tests/utils/tcpClient.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/verification/lib/tests/utils/tcpClient.cpp b/verification/lib/tests/utils/tcpClient.cpp index 5d274d4..a68a407 100644 --- a/verification/lib/tests/utils/tcpClient.cpp +++ b/verification/lib/tests/utils/tcpClient.cpp @@ -50,7 +50,7 @@ uint32_t tcpClient::ipv4_chksum (uint32_t* buf, uint32_t len, bool debug) { uint32_t sum = 0; - for (int idx = 0; idx < len; idx+=2) + for (int idx = 0; idx < (len - 1); idx+=2) { sum += ((buf[idx] & 0xff) << 8) | (buf[idx+1] & 0xff); @@ -542,4 +542,4 @@ uint32_t tcpClient::processFrame (uint32_t* rx_data, uint32_t rx_len) return error; -} \ No newline at end of file +} From 1f609ea42bc5ad1a1334cea1275fb27a2a47cb62 Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 5 Oct 2021 13:38:19 +0100 Subject: [PATCH 06/15] ELGIP-175. Extend sim software (FPM Utils) to be able to support more than 4 Pulse Generators --- verification/lib/tests/utils/fpm6Utils.h | 32 ++++++++++++++---------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/verification/lib/tests/utils/fpm6Utils.h b/verification/lib/tests/utils/fpm6Utils.h index 65fb9ce..a3c83a0 100644 --- a/verification/lib/tests/utils/fpm6Utils.h +++ b/verification/lib/tests/utils/fpm6Utils.h @@ -31,32 +31,38 @@ protected: // ------------------------------------ // Set up FPM for PD emulation // - void configFpmForEmulPulses(CCoreAuto* pCore, + void configFpmForEmulPulses(CCoreAuto* pCore, double encFreqKHz, - double divRatio, - uint32_t numSubPixPhases, - double clkFreqMHz = 100.0, - uint32_t seq[] = NULL, + double divRatio, + uint32_t numSubPixPhases, + double clkFreqMHz = 100.0, + uint32_t seq[] = NULL, uint32_t seqLen = 0, - uint32_t pgSel = 0) + uint32_t pgSel = 0, + CPulseGenerator3Auto* pPulseGenIn = 0) { configPdEmulator (pCore, divRatio, encFreqKHz, clkFreqMHz); - fpmConfig (pCore, divRatio, numSubPixPhases, seq, seqLen, pgSel); + fpmConfig (pCore, divRatio, numSubPixPhases, seq, seqLen, pgSel, pPulseGenIn); }; // ------------------------------------ // Configure the FPM's pulse generator // - int fpmConfig(CCoreAuto* pCore, double divRatio, uint32_t numSubPixPhases, uint32_t seq[], uint32_t seqLen, uint32_t pgSel = 0) + int fpmConfig(CCoreAuto* pCore, double divRatio, uint32_t numSubPixPhases, uint32_t seq[], uint32_t seqLen, uint32_t pgSel = 0, CPulseGenerator3Auto* pPulseGenIn = 0) { CPulseGenerator3Auto* pPulseGen; - switch(pgSel) + if (pPulseGenIn != 0) + pPulseGen = pPulseGenIn; + else { - case 0 : pPulseGen = pCore->pPulseGenerator30; break; - case 1 : pPulseGen = pCore->pPulseGenerator31; break; - case 2 : pPulseGen = pCore->pPulseGenerator32; break; - case 3 : pPulseGen = pCore->pPulseGenerator33; break; + switch(pgSel) + { + case 0 : pPulseGen = pCore->pPulseGenerator30; break; + case 1 : pPulseGen = pCore->pPulseGenerator31; break; + case 2 : pPulseGen = pCore->pPulseGenerator32; break; + case 3 : pPulseGen = pCore->pPulseGenerator33; break; + } } // Program the sequence if a buffer pointer is provided and the length isn't zero. From d3882e025d20c2f3c307757955c1a3a87dfe469b Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 7 Oct 2021 16:43:33 +0100 Subject: [PATCH 07/15] ELGIP-175. Add comment. No functional change --- verification/lib/tests/utils/fpm6Utils.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/verification/lib/tests/utils/fpm6Utils.h b/verification/lib/tests/utils/fpm6Utils.h index a3c83a0..3a8e1bb 100644 --- a/verification/lib/tests/utils/fpm6Utils.h +++ b/verification/lib/tests/utils/fpm6Utils.h @@ -39,7 +39,8 @@ protected: uint32_t seq[] = NULL, uint32_t seqLen = 0, uint32_t pgSel = 0, - CPulseGenerator3Auto* pPulseGenIn = 0) + CPulseGenerator3Auto* pPulseGenIn = 0) // If pPulseGenIn is valid, it overides any selection from pgSel. + // Used to add > 4 Pulse Generators, where required. { configPdEmulator (pCore, divRatio, encFreqKHz, clkFreqMHz); fpmConfig (pCore, divRatio, numSubPixPhases, seq, seqLen, pgSel, pPulseGenIn); From 296b8d3d4e9c2535b242c6982adb11d268af8687 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 11 Nov 2021 08:44:43 +0000 Subject: [PATCH 08/15] Squashed 'autobuild/Common_scripts/' changes from cac10f4..2d042d1 2d042d1 Merge commit '84e6c5693f3652f10fda53d307b924668c5fbdba' into master 1bf6056 Perform a deep copy of version objects b6c8034 Perform a deep copy of version objects e8ea9a2 Perform a deep copy of version objects 907f2d6 Merge commit '3e3fc780b2d0860d3bed5cb788358ebdca90cbfc' into production bc6a637 Submit logging artifacts in the relevant agent environment a14d8b9 Submit logging artifacts in the relevant agent environment c6d435c Change name of repo as its current name is misleading df5d993 Seperate history processing from results 058bfbb Seperate history processing from results 222a237 Move ethernet product script to common scripts area so it can be easily reused fa63436 Do build integrity check and post build actions ad46491 Do build integrity check and post build actions 3f10987 Merge commit '99964dabfc5e2cd5c31875900b728043acd2c178' f368b2c Merge commit '51b0606bac8cc2c19740ce3681a514a95200bd12' into production 428a658 Merge commit 'f233b7565ea64257cc57e9788d47efb84271ea07' into production c31e4a9 Merge commit '583ee7abe5a427cf6e189e7ecfa8ca33772e30c7' into production 58a0ba8 Remove redundant escapes 118a2e2 Remove redundant escapes 9b99cb1 Remove redundant escapes 3b0b793 Remove redundant escapes 8e99ad2 Remove redundant escapes 36c56b1 Remove redundant escapes 1175720 Remove redundant escapes 10a79af Remove redundant escapes 52987df Remove redundant escapes 0c80b05 Do build integrity check and post build actions 00857de Add a new meta walker and test it b9b69d2 Add a new meta walker and test it fe46dfe Add a new meta walker and test it 0f9d832 Add a new meta walker and test it 0073dc7 Add a new meta walker and test it ee7c6e9 Add a new meta walker and test it 7aa361b Add a new meta walker and test it 670ade8 Add a new meta walker and test it e21cbf4 Add a new meta walker and test it 9b773dd Add a new meta walker and test it 11d5325 Add a new meta walker and test it b8a0c96 Add a new meta walker and test it 04067e2 Add a new meta walker and test it e7dace6 Add a new meta walker and test it a89688e Refactor groovy calling test script using newer common script apis; Capture build parameters 6299b7e Refactor groovy calling test script using newer common script apis; Capture build parameters baacfdb Refactor groovy calling test script using newer common script apis; Capture build parameters ea94dfa Refactor groovy calling test script using newer common script apis; Capture build parameters 0c350d8 Refactor groovy calling test script using newer common script apis; Seperate scripts fbd0c16 Refactor groovy calling test script using newer common script apis; Seperate scripts a7f2de8 Refactor groovy calling test script using newer common script apis; Correct history folder 718ebaa Refactor groovy calling test script using newer common script apis; Transfer info across from build; Provide new interface for getting tools d52de4b Trigger various optional processes after a successful build c9e0ee6 Show new tags and their sha code before pushing during release process 0ae743b Check status of jobs 88a4a3b Handle disabled post action jobs; Handle lack of FULLBUILD parameter 9711fd7 Handle disabled post action jobs; Handle lack of FULLBUILD parameter 7fda62f Move post build actions to Common scripts e4bc007 Remove deprecated method 37bf135 Remove redundant condition - try with short unstable builds 9177787 Check for incomplete builds when releasing d86017c Check for incomplete builds when releasing da0ffef Check for incomplete builds when releasing 76ff71a Check for incomplete builds when releasing 6ed4c5d Check for incomplete builds when releasing 0063320 Check for incomplete builds when releasing 2a2b2fa Check for incomplete builds when releasing 4715baa Check for incomplete builds when releasing 21dbda0 Check for incomplete builds when releasing 4925740 Check for incomplete builds when releasing 1d5c4c4 Check for incomplete builds when releasing 7b03667 Check for incomplete builds when releasing 29f72a2 Display cause of build trigger - as the condition for an SCM change is not working 8906c79 Display cause of build trigger - as the condition for an SCM change is not working 7d21209 Display cause of build trigger - as the condition for an SCM change is not working 8332da3 Allow quick builds; Don't let releases proceed if a component is missing artifacts because it is not a full build 02c9115 Merge commit '3c5840b65eb807614455ae25071bfff49f43414b' into production 4d468c3 Allow quick builds; Make quick builds unstable e3a2128 Allow quick builds a21106c Allow quick builds eec1b4a Allow quick builds 349d140 Allow quick builds 024a6b2 Merge commit 'e9d0212c9f667c692d269d5ef21e7401dc3e91e8' 216ab73 Merge commit '7023d604daedb8a50d7c2ab210bcf94186ed4d84' into production 13b84aa Import bootloader binaries into app package 29ef7a0 Merge commit '3151fb17646f8167236d7ba9ba87a23e77b914fa' into production 95ace67 Merge commit '455f7d31d3628a2f5b47617952d36ab18a390350' into master 8f7f366 Merge commit '9ca772699a4a7b4ced7942b5eb888a48b173674b' into master fa5a949 Handle misspelling 6761225 Show which files are being compared 26d4aa2 Correct powershell redirection to null 298c0b5 Merge remote-tracking branch 'origin/production' into master 778d59c Merge commit '15b4f68037538c4be64cdef850bffb9368980122' as 'Common_scripts' 6f51ee8 Remove Common_scripts as this is going to be replaced by a subtree 72b6f55 Use ahb-km-1024a autobuild scripts as starting point 57f0e61 Initial commit ad182bb Merge remote-tracking branch 'origin/production' 264f466 Merge commit '84c862476b3cf1cf22a7c30394b82d5bf9cd0b13' into production 9aa0c94 Merge commit 'cc3196d47dd7ce636bd9344a8eedda1580cb1dd2' into master 18368ff Convert bat to powershell: diffing efa3011 Convert bat to powershell: diffing eed40f2 Convert bat to powershell: diffing 339d0b5 Convert bat to powershell: diffing f0b0a3a Convert bat to powershell: diffing 35e6295 Convert bat to powershell: diffing 4339d74 Convert bat to powershell: diffing 8811df2 Convert bat to powershell: diffing 055270a Convert bat to powershell: diffing 0e5d7b8 Convert bat to powershell: diffing 9f884b1 Convert bat to powershell; Remove debug fa9fb62 Convert bat to powershell; Remove debug 564ed64 Convert bat to powershell; Remove debug b8b6848 Convert bat to powershell; Remove debug efc2ce7 Convert bat to powershell; Remove debug d2e5ef5 Debugging 662e2b7 Debugging 18addad Convert from bat to powershell 07464ef Convert from bat to powershell 4c74502 Debugging northey 149ece2 Debugging northey 6cef1ce Remove debug for northey 3ca4569 Debug northey 2342841 Debug northey 1ca22a8 Debug northey 1c9c8d1 Debug northey 5830e61 Print out git checkout options 6059f51 Debugging why parameter is not making it into script 0a7297e Debugging why parameter is not making it into script b6dd3d5 Merge commit 'bbf885fcff11c0dfb4971925b24cd31b18755b0a' 648fc21 CORE-368 - Place binaries according to associated confluence page 1e531ab CORE-368 - Place binaries according to associated confluence page 9b46b38 Merge branch 'master' of hydra:jenkinsautobuild/Atlas_Autobuild_Scripts into testpipelinedev cfad76f Add checksumming log line 77c41b8 Whitespace; Spelling mistake in comment 8309234 GTS-694: Fix syntax in HMB Test Build e8e5a86 AHBKM1024A-88 - Compare json files in windows 1057c1a AHBKM1024A-88 - Compare json files in windows 7ecd4c7 AHBKM1024A-88 - Compare json files in windows a3da477 AHBKM1024A-88 - Compare json files in windows 24e0140 AHBKM1024A-88 - Compare json files in windows a791e22 AHBKM1024A-88 - Compare json files in windows bf53ffd AHBKM1024A-88 - Compare json files in windows 7fabd5e AHBKM1024A-88 - Compare json files in windows 9e742ea AHBKM1024A-88 - Compare json files in windows 67c272f AHBKM1024A-88 - Compare json files in windows f251bb6 more patchy patchy by extending the timeout for fetch as the repo is apparently huge. 5ae85f4 Update metaexport.groovy e8489e7 Revert temporary changes 82d62ce testing git-subtree-dir: autobuild/Common_scripts git-subtree-split: 2d042d18830183341fc9cca1868667f196c1e696 --- artifacts/metaexport.groovy | 1 - autobuilder.groovy | 78 ++++++++--- build/integrity.groovy | 41 ++++++ build/postactions.groovy | 65 +++++++++ check/build_meta.groovy | 2 +- checksum/utils.groovy | 1 + commandline/commandline.groovy | 3 +- configuration/tools.groovy | 59 ++++++++ copy/all.groovy | 2 +- copy/esw.groovy | 4 +- copy/filtered.groovy | 2 +- copy/logic.groovy | 5 +- differences/file.groovy | 18 ++- differences/json.groovy | 38 +++++- differences/metahierarchy.groovy | 2 +- documentation/doxygen_deploy.groovy | 2 +- meta/folder.groovy | 135 +++++++++++++++++- packaging/debian.groovy | 40 +++--- publish/package.groovy | 2 +- publish/promptingpublish.groovy | 3 +- publish/publish.groovy | 2 +- release/do.groovy | 10 +- release/release.groovy | 2 +- staticanalysis/ccm.groovy | 4 +- staticanalysis/cpd.groovy | 2 +- staticanalysis/cppcheck.groovy | 4 +- test/testethernetproduct.groovy | 203 ++++++++++++++++++++++++++++ versioncontrol/git.groovy | 31 +++-- versionnumbering/do.groovy | 8 +- versionnumbering/generate.groovy | 13 +- 30 files changed, 685 insertions(+), 97 deletions(-) create mode 100644 build/integrity.groovy create mode 100644 build/postactions.groovy create mode 100644 configuration/tools.groovy create mode 100644 test/testethernetproduct.groovy diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 67d13c3..4d93cc5 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,4 +1,3 @@ - // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, importer, subJobs) { def ArrayList artifactFolders = new ArrayList() diff --git a/autobuilder.groovy b/autobuilder.groovy index 2879f11..667f89a 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -43,8 +43,56 @@ def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) { return altered } -// Start up boilderplate -def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String productvariant = '', String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { +def setBuildType(meta, boolean dofullbuild = true) { + def doquickbuild = false + + echo "Build trigger cause: ${currentBuild.getBuildCauses()}" + if (currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause')) { + echo "Build triggered by user" + if (!dofullbuild) { + echo "Marking as quick build as user wanted one" + doquickbuild = true + } + } else { + echo "Build triggered automatically" + if (currentBuild.getBuildCauses('jenkins.branch.BranchEventCause') || currentBuild.getBuildCauses('hudson.triggers.SCMTrigger$SCMTriggerCause')) { + echo "Marking as quick build as it was triggered by a commit" + doquickbuild = true + } else if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) { + echo "Marking as full build as it was triggered by timer" + } + } + echo "Build type is ${doquickbuild ? "quick" : "full"}" + if (doquickbuild) { + unstable('This not a full build - a subset of the artifacts is not being generated') + } + meta.jenkinsbuildinfo.set(!doquickbuild) + return doquickbuild +} + +def setupmeta(script, String productname, String subcomponentname, String productvariant = '') { + def meta = getmetafolderscript().metareset(script) + echo "Product name: ${productname} Subcomponent name: ${subcomponentname} Variant: ${productvariant}" + meta.productinfo.set(productname, subcomponentname, productvariant) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + for (p in params) { + meta.jenkinsbuildinfo.addParameter(p.key, p.value) + } + return meta +} + +// Start up boilerplate +def startbuild( + script, + String productname = PRODUCT_NAME, + String subcomponentname = SUBCOMPONENT_NAME, + String releaserepo = RELEASE_REPO, + String productvariant = '', + String autobuildpath = autobuildroot, + String commonscriptspath = commonscriptsroot + ) { echo "Starting...." autobuildroot = autobuildpath @@ -53,12 +101,7 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam conditionalclean() - def meta = getmetafolderscript().metareset(script) - echo "Product name: ${productname} Subcomponent name: ${subcomponentname}" - meta.productinfo.set(productname, subcomponentname, productvariant) - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + def meta = setupmeta(script, productname, subcomponentname, productvariant) def rr = load("${commonscripts()}/release/registerrepo.groovy") def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant) @@ -72,13 +115,12 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam return meta } -def start(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String jirakey = JIRA_KEY, String releaserepo = RELEASE_REPO, String repoRootSuffix = "", String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) { - echo "DEPRECATED method: start(script, String productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, ...)" - return startbuild(script, productname, subcomponentname, releaserepo, autobuildpath, commonscriptspath) -} - -// Start up boilderplate -def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +// Start up boilerplate +def startnonbuild( + script, + autobuildpath = autobuildroot, + commonscriptspath = commonscriptsroot + ) { echo "Starting non-build job...." autobuildroot = autobuildpath @@ -87,11 +129,7 @@ def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = com conditionalclean() - def meta = getmetafolderscript().metareset(script) - meta.productinfo.set("Non-build-product", "Non-build-subcomponent") - def now = new Date() - now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + def meta = setupmeta(script, "Non-build-product", "Non-build-subcomponent") def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/build/integrity.groovy b/build/integrity.groovy new file mode 100644 index 0000000..73d5d46 --- /dev/null +++ b/build/integrity.groovy @@ -0,0 +1,41 @@ +import static groovy.json.JsonOutput.* + +def check(buildmeta) { + def results = [ + 'good': true, + 'comment': '', + ] + + // Walk the meta looking for different versions of the same components + def allmeta = buildmeta.allrelatedmeta() + echo "All meta:\n${allmeta.comment}" + + def productcomponents = [:] + for (meta in allmeta.metacollection) { + def key = meta.componentidentifier() + + if (!productcomponents.containsKey(key)) { + productcomponents[key] = [:] + } + def buildstampnumber = meta.releasetag() + if (!productcomponents[key].containsKey(buildstampnumber)) { + productcomponents[key][buildstampnumber] = 1 + } else { + productcomponents[key][buildstampnumber] += 1 + } + } + echo prettyPrint(toJson(productcomponents)) + for (pc in productcomponents.keySet()) { + bsns = productcomponents[pc] + if (bsns.size() > 1) { + results.good = false + results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" + for (bsn in bsns.keySet()) { + results.comment += "${bsn}\n" + } + } + } + return results +} + +return this \ No newline at end of file diff --git a/build/postactions.groovy b/build/postactions.groovy new file mode 100644 index 0000000..48ca297 --- /dev/null +++ b/build/postactions.groovy @@ -0,0 +1,65 @@ + +def performSuccessfulBuildActions() { + if (params.DOXYGEN_DEPLOY_JOB != null) { + if (DOXYGEN_DEPLOY_JOB && Jenkins.instance.getItemByFullName(DOXYGEN_DEPLOY_JOB).isBuildable()) { + build( + wait: false, + job: DOXYGEN_DEPLOY_JOB, + parameters: [ + string(name: 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', value: JOB_NAME), + string(name: 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR', value: " ${BUILD_NUMBER}"), + ] + ) + } + } + if (params.STATIC_ANALYSIS_JOB != null) { + if (STATIC_ANALYSIS_JOB && Jenkins.instance.getItemByFullName(STATIC_ANALYSIS_JOB).isBuildable()) { + build( + wait: false, + job: STATIC_ANALYSIS_JOB, + parameters: [ + string(name: 'APPLICATION_ARTIFACT_PROJECT', value: JOB_NAME), + string(name: 'APPLICATION_ARTIFACT_SELECTOR', value: " ${BUILD_NUMBER}"), + ] + ) + } + } + if (params.UNIT_TEST_JOB != null) { + if (UNIT_TEST_JOB && Jenkins.instance.getItemByFullName(UNIT_TEST_JOB).isBuildable()) { + build( + wait: false, + job: UNIT_TEST_JOB, + parameters: [ + string(name: 'APPLICATION_ARTIFACT_PROJECT', value: JOB_NAME), + string(name: 'APPLICATION_ARTIFACT_SELECTOR', value: " ${BUILD_NUMBER}"), + ] + ) + } + } + if (params.REGRESSION_TEST_JOB != null) { + if (REGRESSION_TEST_JOB && Jenkins.instance.getItemByFullName(REGRESSION_TEST_JOB).isBuildable()) { + build( + wait: false, + job: REGRESSION_TEST_JOB, + parameters: [ + string(name: 'APPLICATION_ARTIFACT_PROJECT', value: JOB_NAME), + string(name: 'APPLICATION_ARTIFACT_SELECTOR', value: " ${BUILD_NUMBER}"), + ] + ) + } + } + if (params.RELEASE_TO_GIT_JOB != null) { + if (RELEASE_TO_GIT_JOB && Jenkins.instance.getItemByFullName(RELEASE_TO_GIT_JOB).isBuildable()) { + build( + wait: false, + job: RELEASE_TO_GIT_JOB, + parameters: [ + string(name: 'BUILD_JOB', value: JOB_NAME), + string(name: 'BUILD_JOB_SELECTOR', value: " ${BUILD_NUMBER}"), + ] + ) + } + } +} + +return this; diff --git a/check/build_meta.groovy b/check/build_meta.groovy index 34eed98..6e669bc 100644 --- a/check/build_meta.groovy +++ b/check/build_meta.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/checksum/utils.groovy b/checksum/utils.groovy index c2503cd..be7569d 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -5,6 +5,7 @@ def generatelist(meta, checksummable) { def checksumset = meta.checksuminfo.addChecksumSet(filepath) def files = findFiles(glob: filepath) for (file in files) { + echo "Checksumming: ${file.path}" def sum = sha1(file: file.path).trim() checksumset.addChecksumPair(file.path, sum) } diff --git a/commandline/commandline.groovy b/commandline/commandline.groovy index df99a75..38f80fa 100644 --- a/commandline/commandline.groovy +++ b/commandline/commandline.groovy @@ -2,8 +2,7 @@ String getCommandOutput(String cmd) { if (isUnix()) { return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() } else { - return bat(returnStdout: true, script: """ - @echo off + return powershell(returnStdout: true, script: """ ${cmd} """ ).trim() diff --git a/configuration/tools.groovy b/configuration/tools.groovy new file mode 100644 index 0000000..dbaeedb --- /dev/null +++ b/configuration/tools.groovy @@ -0,0 +1,59 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +class ToolOptions { + def m_script // Script to use if wanting to execute script commands (eg. echo -> script.echo) + + String m_toolName + + String m_gitURL + String m_revision + String m_repoRoot + int m_cloneDepth + boolean m_shallowClone + + ToolOptions(script) { + m_script = script + m_gitURL = '-' + m_revision = '-' + m_repoRoot = '' + m_cloneDepth = 1 + m_shallowClone = true + } + + void toolName(String name) { + m_toolName = name.trim() + } + + void toolRepo(String gitURL, String revision) { + m_gitURL = gitURL.trim() + m_revision = revision.trim() + } + + void repoRoot(String root) { + m_repoRoot = root.trim() + } + + void clonedepth(int depth) { + m_cloneDepth = depth + } + + void shallowclone(boolean shallow) { + m_shallowClone = shallow + } +} + +ToolOptions getNewToolOptions() { + return new ToolOptions(this) +} + +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) +} + +def getTool(meta, ToolOptions options, String targetDir) { + gh.gitCheckoutAlt(options.m_gitURL, options.m_revision, targetDir, options.m_repoRoot, options.m_cloneDepth, options.m_shallowClone) + + registerConfigurationInfo(meta, options.m_toolName, options.m_gitURL, options.m_revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/copy/all.groovy b/copy/all.groovy index b7c4607..6d09903 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/copy/esw.groovy b/copy/esw.groovy index 2a63171..c6c0147 100644 --- a/copy/esw.groovy +++ b/copy/esw.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -41,7 +41,7 @@ pipeline { rm -v -rf meta mv -v "${ESW}"/* . """ - archiveArtifacts "${artifactlist}" + archiveArtifacts artifactlist } } } diff --git a/copy/filtered.groovy b/copy/filtered.groovy index feaa2ab..7fd87b6 100644 --- a/copy/filtered.groovy +++ b/copy/filtered.groovy @@ -48,7 +48,7 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["filtered/**"]) - archiveArtifacts "${artifactlist}" + archiveArtifacts artifactlist } } } diff --git a/copy/logic.groovy b/copy/logic.groovy index 7ae0fe1..9585f01 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -26,6 +26,7 @@ pipeline { stage('Copying logic artifacts') { steps { script { + sh('env') def logicoptions = importer.getNewImportOptions() logicoptions.source(LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_ARTIFACT_RELEASE_REPO, LOGIC_ARTIFACT_RELEASE_REVISION) importer.artifactimport(meta, logicoptions, LOGICDEST) @@ -42,7 +43,7 @@ pipeline { rm -v -rf diff mv -v "logic"/* . """ - archiveArtifacts "${artifactlist}" + archiveArtifacts artifactlist } } } diff --git a/differences/file.groovy b/differences/file.groovy index ab4102f..ee1b8b3 100644 --- a/differences/file.groovy +++ b/differences/file.groovy @@ -1,15 +1,23 @@ def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { + echo "Comparing file: ${importedfile} with: ${currentfile} option: ${ignorewhitespace}" + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } if (isUnix()) { - def wsparameter = "-w" - if (!ignorewhitespace) { - wsparameter = "" - } sh """ diff -c ${wsparameter} "${importedfile}" "${currentfile}" """ } else { - error "Windows not currently supporting file comparison" + // diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # where diff + # if errorlevel 1 EXIT /b 1 + + diff.exe -c ${wsparameter} "${importedfile}" "${currentfile}" + """ } } diff --git a/differences/json.groovy b/differences/json.groovy index 5ac37f2..dbf47b8 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -1,4 +1,5 @@ def comparefiles(importedjsonfile, currentjsonfile) { + echo "Comparing json file: ${importedjsonfile} with: ${currentjsonfile}" if (isUnix()) { sh """ which jq @@ -8,7 +9,42 @@ def comparefiles(importedjsonfile, currentjsonfile) { """ } else { - error "Windows not currently supporting json file comparison" + // Jq and diffutils can be installed on windows with chocolatey + powershell """ + # echo Checking utilities exist + # set + # echo %PATH% + try { + if (Get-Command jq) {} + } + catch { + echo "jq is unavailable" + exit \$false + } + try { + if (Get-Command diff) {} + } + catch { + echo "diff is unavailable" + exit \$false + } + + # echo Sorting json in files to compare + # jq -S sorts json output + jq -S . "${importedjsonfile}" > temp1.txt + if (\$lastexitcode -ge 1) { exit 1 } + jq -S . "${currentjsonfile}" > temp2.txt + if (\$lastexitcode -ge 1) { exit 1 } + + # echo Diffing sorted json files + diff.exe -c temp1.txt temp2.txt + \$lec = \$lastexitcode + + Remove-Item temp1.txt + Remove-Item temp2.txt + + if (\$lec -ge 1) { exit 1 } + """ } } diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 4bcdcdc..10b24a0 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/documentation/doxygen_deploy.groovy b/documentation/doxygen_deploy.groovy index 6642256..5054c02 100644 --- a/documentation/doxygen_deploy.groovy +++ b/documentation/doxygen_deploy.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/meta/folder.groovy b/meta/folder.groovy index a2fd4e1..aee84f6 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -38,7 +38,7 @@ class ProductInfo extends GISJsonSerialised { pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - pstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${productname}\" > NUL 2> NUL", , returnStatus: true) + pstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${productname}\" 2>&1>\$null", , returnStatus: true) } if (pstatus > 0) { script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") @@ -48,7 +48,7 @@ class ProductInfo extends GISJsonSerialised { sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - sstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${subcomponentname}\" > NUL 2> NUL", , returnStatus: true) + sstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" 2>&1>\$null", , returnStatus: true) } if (sstatus > 0) { script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") @@ -59,7 +59,7 @@ class ProductInfo extends GISJsonSerialised { vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true) } else { - vstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${variant}\" > NUL 2> NUL", , returnStatus: true) + vstatus = script.powershell(script: "git check-ref-format --normalize \"refs/tags/${variant}\" 2>&1>\$null", , returnStatus: true) } if (vstatus > 0) { script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name") @@ -70,6 +70,12 @@ class ProductInfo extends GISJsonSerialised { this.variant = variant } + void set(ProductInfo otherinfo) { + this.productname = otherinfo.productname + this.subcomponentname = otherinfo.subcomponentname + this.variant = otherinfo.variant + } + void deserialiseJson(rawmeta) { productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] @@ -102,6 +108,15 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + GitInfo(GitInfo another) { + super(another.script) + this.sourcepath = another.sourcepath + this.sourcerevision = another.sourcerevision + this.gitrevision = another.gitrevision + this.remoteurl = another.remoteurl + this.remoteinfo = another.remoteinfo + } + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { this.sourcepath = sourcepath this.sourcerevision = sourcerevision @@ -110,6 +125,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + void set(GitInfo otherinfo) { + this.sourcepath = otherinfo.sourcepath + this.sourcerevision = otherinfo.sourcerevision + this.gitrevision = otherinfo.gitrevision + this.remoteurl = otherinfo.remoteurl + this.remoteinfo = otherinfo.remoteinfo + } + void deserialiseJson(rawmeta) { sourcepath = rawmeta['sourcepath'] sourcerevision = rawmeta['sourcerevision'] @@ -130,25 +153,78 @@ class GitInfo extends GISJsonSerialised { } } +class JenkinsBuildParameter extends GISJsonSerialised { + private String name + private String valuetype + private String value + + JenkinsBuildParameter(script, String name = '', String valuetype = '', String value = '') { + super(script) + this.name = name + this.valuetype = valuetype + this.value = value + } + + void set(String name, String valuetype, String value) { + this.name = name + this.valuetype = valuetype + this.value = value + } + + void deserialiseJson(rawmeta) { + name = rawmeta['name'] + valuetype = rawmeta['valuetype'] + value = rawmeta['value'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'name' : name, + 'valuetype' : valuetype, + 'value' : value, + ] + return data + } +} + class JenkinsBuildInfo extends GISJsonSerialised { String buildjobname int buildnumber private String buildmachine private String builddate + ArrayList parameters + boolean isfullbuild - JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "", boolean isfullbuild = true) { super(script) this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine this.builddate = builddate + this.isfullbuild = isfullbuild + parameters = new ArrayList() } - void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + void set(String buildjobname, int buildnumber, String buildmachine, String builddate, boolean isfullbuild = true) { this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine this.builddate = builddate + this.isfullbuild = isfullbuild + } + + void set(boolean isfullbuild) { + this.isfullbuild = isfullbuild + } + + boolean isFullBuild() { + return isfullbuild + } + + void addParameter(String name, value) { + def p = new JenkinsBuildParameter(script, name, value.getClass().toString(), value.toString()) + parameters.add(p) + return p } void deserialiseJson(rawmeta) { @@ -156,6 +232,23 @@ class JenkinsBuildInfo extends GISJsonSerialised { buildnumber = rawmeta['buildnumber'] buildmachine = rawmeta['buildmachine'] builddate = rawmeta['builddate'] + isfullbuild = (rawmeta['isfullbuild'] == null) ? true : rawmeta['isfullbuild'] + parameters = new ArrayList() + if (rawmeta['isfullbuild'] != null) { + for (rawparameter in rawmeta['parameters']) { + def p = new JenkinsBuildParameter(script) + p.deserialiseJson(rawparameter) + parameters.add(p) + } + } + } + + List parametersserialised() { + def associated = [] + for (p in parameters) { + associated.add(p.serialisationdata()) + } + return associated } LinkedHashMap serialisationdata() { @@ -164,6 +257,8 @@ class JenkinsBuildInfo extends GISJsonSerialised { 'buildnumber' : buildnumber, 'buildmachine' : buildmachine, 'builddate' : builddate, + 'isfullbuild' : isfullbuild, + 'parameters' : parametersserialised(), ] return data } @@ -181,6 +276,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstamp = buildstamp } + BuildStampInfo(BuildStampInfo another) { + super(another.script) + this.buildstampprefix = another.buildstampprefix + this.revision = another.revision + this.buildstamp = another.buildstamp + } + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision @@ -539,6 +641,10 @@ class MetaInfo extends GISJsonSerialised { return custominfo.getAt(customId.trim()) } + def isFullBuild() { + return jenkinsbuildinfo.isFullBuild() + } + List publishableinfoserialised() { def publishable = [] for (publishableitem in publishableinfo) { @@ -762,6 +868,25 @@ class MetaInfo extends GISJsonSerialised { return releasemetaversionuniquecollection } + def walkrelated(ArrayList metacollection, String comment = '', int level = 0) { + def results = [ + 'metacollection' : metacollection, + 'comment': comment + ] + results.metacollection.add(this) + results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + results = othermeta.walkrelated(results.metacollection, results.comment, level + 1) + } + return [ 'metacollection' : results.metacollection, 'comment': results.comment ] + } + + def allrelatedmeta() { + ArrayList metacollection = new ArrayList() + return walkrelated(metacollection) + } + MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } diff --git a/packaging/debian.groovy b/packaging/debian.groovy index ead24fa..c0a4dbb 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -20,6 +20,7 @@ class DebRootDefinition { String GIS_OPT_KERNEL String GIS_OPT_LOGIC String GIS_OPT_PRINTHEAD + String GIS_OPT_BOOT String USR_BIN String USR_LIB_ARCH @@ -27,7 +28,7 @@ class DebRootDefinition { String HTTP_DEV_DIV String LIB_SYSTEMD - DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp) { + DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix) { ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix @@ -38,28 +39,29 @@ class DebRootDefinition { PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" - ETC = "${DEBPATH}/etc" - GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" - GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" - GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" - GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" - GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" - GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" - GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" - GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" - GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" - GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + ETC = "${DEBPATH}/etc" + inpackagepathsuffix + GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + inpackagepathsuffix + GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + inpackagepathsuffix + GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + inpackagepathsuffix + GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + inpackagepathsuffix + GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" + inpackagepathsuffix + GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + inpackagepathsuffix + GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + inpackagepathsuffix + GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel" + inpackagepathsuffix + GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + inpackagepathsuffix + GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + inpackagepathsuffix + GIS_OPT_BOOT = "${DEBPATH}/opt/gis/boot" + inpackagepathsuffix - USR_BIN = "${DEBPATH}/usr/bin" - USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" - LIB_FIRMWARE = "${DEBPATH}/lib/firmware" - HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" - LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + USR_BIN = "${DEBPATH}/usr/bin" + inpackagepathsuffix + USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + inpackagepathsuffix + LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + inpackagepathsuffix + HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + inpackagepathsuffix + LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + inpackagepathsuffix } } -def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp) { - return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp) +def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp, String inpackagepathsuffix = '') { + return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) } def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { diff --git a/publish/package.groovy b/publish/package.groovy index 3bd9565..958a614 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 4af0d62..9d51915 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -62,6 +62,7 @@ pipeline { echo("Showing filtered releases") for (release in allreleases) { if (!release.toLowerCase().startsWith("superceded") && + !release.toLowerCase().startsWith("superseded") && !release.toLowerCase().startsWith("erroneous") && !release.toLowerCase().startsWith("faulty") && !release.toLowerCase().startsWith("ignore") && diff --git a/publish/publish.groovy b/publish/publish.groovy index 7beb134..82a32ad 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/release/do.groovy b/release/do.groovy index 9148572..e0b9471 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -235,6 +235,7 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) summaryappend("Rel Tag ") for (tag in taglist) { if (releasegittag(postcommitshacode, tag)) { + echo "Tag added: ${tag} with sha: ${postcommitshacode}" tagadded = true } } @@ -420,7 +421,14 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { - error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" + } + } + + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + if (!buildinfo.jenkinsbuildinfo.isFullBuild()) { + error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" } } diff --git a/release/release.groovy b/release/release.groovy index 4220f3b..351985d 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index f0e5b80..5a7c6f5 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -97,7 +97,7 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + archiveArtifacts artifactlist } } } diff --git a/staticanalysis/cpd.groovy b/staticanalysis/cpd.groovy index 6f3a33f..5a77cab 100644 --- a/staticanalysis/cpd.groovy +++ b/staticanalysis/cpd.groovy @@ -129,7 +129,7 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + archiveArtifacts artifactlist } } } diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index bdc6a2a..9455b0b 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -1,5 +1,5 @@ pipeline { - agent { label "${AGENT_LABEL}" } + agent { label AGENT_LABEL } stages { stage('Initial setup') { steps { @@ -91,7 +91,7 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts "${artifactlist}" + archiveArtifacts artifactlist } } } diff --git a/test/testethernetproduct.groovy b/test/testethernetproduct.groovy new file mode 100644 index 0000000..92d6414 --- /dev/null +++ b/test/testethernetproduct.groovy @@ -0,0 +1,203 @@ +import jenkins.model.Jenkins + +pipeline { + agent { label AGENT_LABEL } + stages { + stage ("Initial Setup") { + steps{ + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage ("Checkout test environment") { + steps { + script { + toolsimporter = load("${autobuilder.commonscripts()}/configuration/tools.groovy") + + def TEST_ENVIRONMENT_URL = 'git@hydra:gistest/jenkins-ethernet-test-environment.git' + def TEST_ENVIRONMENT_DIR = 'TestEnvironmentDir' + def TEST_ENVIRONMENT_REVISION = 'origin/master' + + def environmentoptions = toolsimporter.getNewToolOptions() + environmentoptions.toolName('Ethernet product test environment') + environmentoptions.toolRepo(TEST_ENVIRONMENT_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, environmentoptions, TEST_ENVIRONMENT_DIR) + + TEST_ENVIRONMENT_FILE = "${TEST_ENVIRONMENT_DIR}/environment.json" + def alltestenvironment = readJSON(file: "${TEST_ENVIRONMENT_FILE}") + TEST_ENVIRONMENT = alltestenvironment[ETHERNET_PRODUCT_TO_RUN_TEST] + echo "Selected environment:\n${TEST_ENVIRONMENT}" + } + } + } + stage ('Setup') { + agent { label TEST_ENVIRONMENT['FTDI_AGENT'] } + stages { + stage ("Checkout dependencies") { + steps { + script { + def PYTHON_URL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHON_REVISION = 'origin/3.6.3_linux' + } + else { + PYTHON_REVISION = 'origin/3.8.10_windows_for_test' + } + PYTHONDIR = "${WORKSPACE}/python" + + def pythonimportoptions = toolsimporter.getNewToolOptions() + pythonimportoptions.toolName('Python') + pythonimportoptions.toolRepo(PYTHON_URL, PYTHON_REVISION) + toolsimporter.getTool(meta, pythonimportoptions, PYTHONDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + + def TEST_SOURCE_URL = 'git@hydra:hmb_regression/hmb_tests.git' + TEST_SUITE_DIR = 'TestSuiteDir' + env.GIS_TEST_SUITE_HOME = "${WORKSPACE}/${TEST_SUITE_DIR}" + + def regressiontestimportoptions = toolsimporter.getNewToolOptions() + regressiontestimportoptions.toolName('Tests') + regressiontestimportoptions.toolRepo(TEST_SOURCE_URL, TEST_SOURCE_REVISION) + toolsimporter.getTool(meta, regressiontestimportoptions, GIS_TEST_SUITE_HOME) + + env.HISTORY_FOLDER = "${WORKSPACE}/testhistory" + + def resultimportoptions = toolsimporter.getNewToolOptions() + resultimportoptions.toolName('Test history') + resultimportoptions.toolRepo(HISTORY_REPO, HISTORY_REVISION) + toolsimporter.getTool(meta, resultimportoptions, HISTORY_FOLDER) + + def CORE_SOURCE_URL = 'git@hydra:gistest/giscorelite.git' + env.GIS_TEST_SUITE_CORE = "${WORKSPACE}/GisCore" + + def coreimportoptions = toolsimporter.getNewToolOptions() + coreimportoptions.toolName('Test core') + coreimportoptions.toolRepo(CORE_SOURCE_URL, CORE_SOURCE_REVISION) + toolsimporter.getTool(meta, coreimportoptions, GIS_TEST_SUITE_CORE) + + def FTDI_SOURCE_URL = 'git@hydra:python_scripts/ftdi_relay.git' + def FTDI_SOURCE_REVISION = 'origin/Core-98' + FTDI_SOURCE_DIR = "${WORKSPACE}/PowerController" + + def ftdiimportoptions = toolsimporter.getNewToolOptions() + ftdiimportoptions.toolName('FTDI relay driver') + ftdiimportoptions.toolRepo(FTDI_SOURCE_URL, FTDI_SOURCE_REVISION) + toolsimporter.getTool(meta, ftdiimportoptions, FTDI_SOURCE_DIR) + + def HISTORY_PROCESSING_REPO = 'git@hydra:gistestresults/ethernet-products-history-processing.git' + HISTORY_PROCESSING_DIR = "${WORKSPACE}/HistoryProcessing" + + def resultprocessingoptions = toolsimporter.getNewToolOptions() + resultprocessingoptions.toolName('History processing') + resultprocessingoptions.toolRepo(HISTORY_PROCESSING_REPO, HISTORY_PROCESSING_REVISION) + toolsimporter.getTool(meta, resultprocessingoptions, HISTORY_PROCESSING_DIR) + } + } + } + stage ("Import application to test") { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + APPLICATION_UNDER_TEST = 'application_under_test' + env.APPLICATION_ARTIFACT_FOLDER = "${WORKSPACE}/${APPLICATION_UNDER_TEST}" + + def applicationimportoptions = importer.getNewImportOptions() + applicationimportoptions.jenkinsjob(APPLICATION_ARTIFACT_PROJECT, APPLICATION_ARTIFACT_SELECTOR) + applicationimportoptions.setfilter('app_package/**,control_package/**') + applicationmeta = importer.artifactimport(meta, applicationimportoptions, APPLICATION_ARTIFACT_FOLDER) + + meta.productinfo.set(applicationmeta.productinfo) + meta.productinfo.subcomponentname = "${applicationmeta.productinfo.subcomponentname}_TEST" + meta.gitinfo.set(applicationmeta.gitinfo) + meta.buildstampinfo.set(applicationmeta.buildstampinfo) + } + } + } + stage("Clean") { + steps { + script { + LOGGING_ROOT = 'Logging' + env.GIS_TEST_OUTPUT_LOGGING_ROOT = "${WORKSPACE}/${LOGGING_ROOT}" + cleanfolders = [ + GIS_TEST_OUTPUT_LOGGING_ROOT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage ('Test') { + steps { + script { + lock(TEST_ENVIRONMENT['RESOURCE_LOCK']) { + if (POWER_CYCLE_HARDWARE == 'true') { + dir(FTDI_SOURCE_DIR) { + bat """ + "${PYTHON}" "${FTDI_SOURCE_DIR}/RegressionPower.py" -d ${TEST_ENVIRONMENT['FTDI_RELAY_DEVICE']} -o True + """ + } + } + if (UPDATE_HARDWARE == 'true') { + dir(GIS_TEST_SUITE_CORE) { + bat """ + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbPackageDeployer.py" -p "${APPLICATION_ARTIFACT_FOLDER}" -t "${TEST_ENVIRONMENT['IP_ADDRESS']}" + """ + rc = bat (script:""" + "${PYTHON}" "${GIS_TEST_SUITE_CORE}/HMBCommon/hmbinspectstate.py" + """, + returnStatus: true) + if (rc == 1) { + unstable('FAILED while inspecting state of hardware') + } + } + } + if (DO_TEST == 'true') { + rc = bat(script:""" + ${PYTHON} "${GIS_TEST_SUITE_CORE}/GISTestMaster.py" -w ${PRODUCT_LOCATION_UNDER_TEST} -fl --include ${TEST_INCLUDE_FILTER} + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED while running tests') + } + } + } + } + } + } + stage("Archive Test History") { + steps{ + script { + dir(HISTORY_FOLDER) { + rc = bat (script:""" + "${PYTHON}" "${HISTORY_PROCESSING_DIR}/GitCyclicPush.py" + """, + returnStatus: true + ) + if (rc == 1) { + unstable('FAILED to process history') + } + } + } + } + } + stage("Archive test artifacts"){ + steps{ + script { + def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) + archiveArtifacts artifactlist + } + } + } + } + } + } +} + diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f5b2ebd..50f4025 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -12,7 +12,7 @@ def boolean isShallowClonedRepo(String gitrootpath) { isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() } else { - isshallowclone = bat(returnStdout: true, script: "@git rev-parse --is-shallow-repository").trim().toBoolean() + isshallowclone = powershell(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() } echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" } @@ -89,9 +89,9 @@ def String checkouttoreporoot(String gitURL) { if [ \$? -ne 0 ]; then exit 1; fi """ } else { - bat """ + powershell """ git status - if errorlevel 1 EXIT /b 1 + if (\$lastexitcode -ge 1) { exit 1 } """ } } @@ -109,9 +109,9 @@ def checkgitlfsinstall() { """ } else { - bat """ + powershell """ git lfs install - if errorlevel 1 EXIT /b 1 + if (\$lastexitcode -ge 1) { exit 1 } """ } } @@ -185,6 +185,7 @@ def showsparseitemlist(ArrayList sparseitemlist = new ArrayList( } def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { + echo "Git: Checking out: URL: ${gitURL} Revision: ${revision} to: ${targetDir}" checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -195,8 +196,8 @@ def docheckout(String gitURL, String revision, int cloneDepth, String reporootpa ], doGenerateSubmoduleConfigurations: false, extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], + [$class: 'GitLFSPull',timeout: 25], + [$class: 'CleanBeforeCheckout',timeout: 20], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], @@ -221,7 +222,7 @@ def gitinit(String targetDir, String reporootpath) { git init "${targetDir}" """ } else { - bat """ + powershell """ git init "${targetDir}" """ } @@ -245,7 +246,7 @@ def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { git config lfs.fetchinclude "${sparseitemliststring}" """ } else { - bat """ + powershell """ git config lfs.fetchinclude "${sparseitemliststring}" """ } @@ -263,10 +264,10 @@ def gitsparserestore(String targetDir) { git reset --hard > /dev/null """ } else { - bat """ - git rm -rf . 2> nul - git clean -fxd 2> nul - git reset --hard 2> nul + powershell """ + git rm -rf . > \$null + git clean -fxd > \$null + git reset --hard > \$null """ } } @@ -311,8 +312,8 @@ def checkRepoExists(String gitURL) { returnStatus: true ) } else { - result = bat( script: """ - @git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + result = powershell( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY """, returnStatus: true ) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 666aee3..208a60f 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -4,17 +4,17 @@ def stamp(convertdetails) { def filetoconvert = convertdetail[0] def pattern = convertdetail[1] def replacement = convertdetail[2] - //echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - //echo "Using python: ${PYTHON} and script: ${stampscript}" + echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } else { - bat """ - "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + powershell """ + & "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 71d7bf6..6e0db11 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -28,7 +28,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } else { - changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() + changecount = powershell(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" def int mvnp1 = maxversionnumber() + 1 @@ -39,21 +39,21 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() } else { - gitrevision = bat(returnStdout: true, script: "@git rev-parse HEAD").trim() + gitrevision = powershell(returnStdout: true, script: "git rev-parse HEAD").trim() } echo "Revision: ${gitrevision}" if (isUnix()) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() } else { - remoteinfo = bat(returnStdout: true, script: "@git remote show origin").trim() + remoteinfo = powershell(returnStdout: true, script: "git remote show origin").trim() } echo "Remote info: ${remoteinfo}" if (isUnix()) { remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() } else { - remoteurl = bat(returnStdout: true, script: "@git config --get remote.origin.url").trim() + remoteurl = powershell(returnStdout: true, script: "git config --get remote.origin.url").trim() } echo "Remote URL: ${remoteurl}" } @@ -160,8 +160,9 @@ String generate(meta, String sourcerevision, String sourcepath, String basebuild } String copyfrom(meta, othermeta) { - meta.gitinfo = othermeta.gitinfo - meta.buildstampinfo = othermeta.buildstampinfo + // Deep copy + meta.gitinfo.set(othermeta.gitinfo.sourcepath, othermeta.gitinfo.sourcerevision, othermeta.gitinfo.gitrevision, othermeta.gitinfo.remoteurl, othermeta.gitinfo.remoteinfo) + meta.buildstampinfo.set(othermeta.buildstampinfo.buildstampprefix, othermeta.buildstampinfo.revision, othermeta.buildstampinfo.buildstamp) return meta.buildstampinfo.buildstamp } From d030a01e3c152e18ae04dcdf7c261c7386e3ed72 Mon Sep 17 00:00:00 2001 From: Dave Steele Date: Fri, 10 Dec 2021 14:26:11 +0000 Subject: [PATCH 09/15] CORE-580 Change Jenkins to store full Altera build files as Artifacts --- autobuild/synthesis/synthesis.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuild/synthesis/synthesis.gsh b/autobuild/synthesis/synthesis.gsh index 34618db..dec7fd8 100644 --- a/autobuild/synthesis/synthesis.gsh +++ b/autobuild/synthesis/synthesis.gsh @@ -170,7 +170,7 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["embedded_logic/**"]) + def artifactlist = meta.generateartifacts(["embedded_logic/**","el/firmware/**"]) archiveArtifacts "${artifactlist}" } } From 854d51d9ad0caf0dabdcb1eb2adf73d716147f65 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Jan 2022 10:20:21 +0000 Subject: [PATCH 10/15] Squashed 'autobuild/Common_scripts/' changes from 2d042d1..6f0a448 6f0a448 Handle old components that have blank product and component names bf60873 Show which component is being released 9c400e8 Merge commit '7fa33cb2cbb0ab652f4e01f10b32ad0c9f00bb67' into production 79c1928 Remove debug from console 8034fe0 Merge commit '7fb8ecb7e0e28ce1e6715a7b2899f83b6a3491e1' 600d458 Remove timeout warning 8297893 Correct publish list fb35654 Exclude erroneous sub-component from release c8a66b7 Exclude erroneous sub-component from release bdbc579 Pass meta path to publisher walker e5e53b3 Merge commit '01aa63549946d53c17612f7388e77bc5884439c0' into production e875175 Correct diffing to use new meta hierarchy 83e7f69 Merge commit '0728204dc5a244ce2d9496d8c6fbb5fb2ab58d07' into production 4663850 Correct walking through v0.1 associatedmeta aad347e Merge commit '0e3ea698c71c1d697dfeab5c9a2d6010a1ee4d9e' 9278e5a Merge commit 'c6e4ee3453e4cbed98b0573a569d79f1e5cdf19d' 85dd653 Optimise reading in of meta from files 984d996 Optimise reading in of meta from files 88ca474 Optimise reading in of meta from files d7965ea Optimise reading in of meta from files cfa5439 Optimise reading in of meta from files a2ebb96 Optimise reading in of meta from files f21873b Optimise reading in of meta from files 27298b9 Use an iteration variable in walker 904e2cc Use an iteration variable in walker 345d6f2 Use an iteration variable in walker fd8a382 Use an iteration variable in walker 72b5f6d Use an iteration variable in walker 0122ee4 Use an iteration variable in walker bfb7d9f Use an iteration variable in walker 902ec65 Use an iteration variable in walker 71a3a92 Add a unique id to each product ebb151d Add a unique id to each product fb4e313 Merge commit '8470480c48645f659ec2753f002fc26adb6e07f2' 14adb01 Merge commit '73f4794d6bbe777cbe4d46ae0f52befe46c73249' into production 4b3e84c Alter path to start finding af70f7e Add log d5ecbe1 Add log 6053dbc Correct initial gather paths f171e52 Correct initial gather paths 09b3c99 Correct initial gather paths d3e5eb9 Correct build integrity checking 01d66c2 Don't create folders for imported artifacts that don't have meta c9eb0c6 Don't create folders for imported artifacts that don't have meta d7893d9 Use clean while moving imported info a06ac84 Use clean while moving imported info c3c7b80 Use clean while moving imported info 6b35583 Use clean while moving imported info 07a8a9d Use indirect artifact archiver ab0d724 Load module before cleaning 0a0df4d Load module before cleaning 1a11141 Type parameters to archive method 8e53968 Type parameters to archive method 670d7b2 Handle v0.1 imported associated meta c76a1d6 Handle v0.1 imported associated meta 25938ce Handle v0.1 imported associated meta 95506b8 Handle v0.1 imported associated meta 7e63a31 Handle v0.1 imported associated meta 6e3c993 Handle v0.1 imported associated meta cd43ecf Handle v0.1 imported associated meta f44dc09 Handle v0.1 imported associated meta f09c84e Handle v0.1 imported associated meta d93eb4b Handle v0.1 imported associated meta 4005488 Handle v0.1 imported associated meta 16061cf Help the meta walker find the meta data for a child f554d5b Help the meta walker find the meta data for a child d945efb Help the meta walker find the meta data for a child 2a7e94b Debug components 5e1a008 Alter release process to handle new meta walker info 29f6660 Alter release process to handle new meta walker info 9d4b6ec Alter release process to handle new meta walker info 236aa6f Alter release process to handle new meta walker info 8ccca6a Alter release process to handle new meta walker info 2f8ac67 Alter release process to handle new meta walker info af1c499 Alter release process to handle new meta walker info 2fe1414 Alter release process to handle new meta walker info 0b057d0 Alter release process to handle new meta walker info 206e0f3 Alter release process to handle new meta walker info a342a0f Alter release process to handle new meta walker info 55a087b Make the walkers carry more meta data dcd5fe6 Make the walkers carry more meta data 4b72bba Make the walkers carry more meta data 61e2834 Make the walkers carry more meta data 93ad1be Make the walkers carry more meta data 053b39b Make the walkers carry more meta data 5b50023 Merge remote-tracking branch 'origin/production' 755bb99 Wipe out an additional folder that the logic is now exporting ece2670 Make the walkers carry more meta data b973208 Try to walk through build meta items 99aa54a Try to walk through build meta items 5b35002 Try to walk through build meta items 82dc66d Use importer to get build artifacts to release f417682 Use importer to get build artifacts to release b64672c Use importer to get build artifacts to release e35826f Use importer to get build artifacts to release 135fddb Use importer to get build artifacts to release fc34f64 Use importer to get build artifacts to release f3a0f9f Use importer to get build artifacts to release aff9e3e Use importer to get build artifacts to release 96621af Use importer to get build artifacts to release 7d8e910 Use importer to get build artifacts to release 1ee2be6 Use importer to get build artifacts to release 9b27520 Make all walkers use a walker that understand v0.1 and v0.2 formats c985dcb Use non-deprecated methods for importing 523407f Make sure all jobs are using the new archiver 149e1fd Make sure all jobs are using the new archiver e9157c3 Make sure all jobs are using the new archiver e57708a Make sure all jobs are using the new archiver 6502cf7 Make sure all jobs are using the new archiver 0406350 Make sure all jobs are using the new archiver 9e0fe09 Make sure all jobs are using the new archiver f31914d Make sure all jobs are using the new archiver 2ba3717 Handle v0.1 and v0.2 formats simultaneously a646881 Remove debug; RelatedMetaInfo does not store meta info anymore 6bf8b58 Walk a tree of imported meta 0359773 Walk a tree of imported meta af5f19b Walk a tree of imported meta 04d17c1 Walk a tree of imported meta 84c313a Walk a tree of imported meta 3b446d0 Walk a tree of imported meta 6200503 Walk a tree of imported meta 3447179 Walk a tree of imported meta 0d396e0 Walk a tree of imported meta a87cb8b Walk a tree of imported meta 71349d9 Walk a tree of imported meta 4132c4d Walk a tree of imported meta c39aac8 Use credential manager to perform sudo operations c7ff759 Use credential agent to supply password 36b9f2e Use credential agent to supply password b397a07 Remove python checkout when it is not used 450f38e Relay input meta files to output meta folders e466480 Relay input meta files to output meta folders f98f9e9 Relay input meta files to output meta folders dfcbbf1 Record the import path of meta data 19d7103 Don't serialise associated meta data 0701945 Start writing format version into meta data file df492c8 Start writing format version into meta data file c661492 Start writing format version into meta data file 74ad66b Start writing format version into meta data file c916bf9 Merge remote-tracking branch 'origin/master' into production 9aeb686 erge commit '26c90cfbe72aa574c4df627e618e5ab2a230a41b' into production 557ad6c Merge commit '1c5d8fc0fd06e72f20600f51680bbe629f010b82' into master e75eb96 Handle long version number strings eg. in sha codes c989904 Add a default archiver cb65d2b Add a default archiver 8c2b3ab Add a default archiver 8d3afbb Add a default archiver a301529 Add a default archiver 0602dae Add a default archiver 0f217b8 Add a default archiver a91d0d5 Add a default archiver 2763813 Add a default archiver 721375a Defer archiving artifacts to a loaded script 710557d Handle pre-generated buildnumber 603e13c Handle pre-generated buildnumber 79d92bf Handle pre-generated buildnumber 7b1eb11 Handle pre-generated buildnumber 9246812 Handle pre-generated buildnumber d261eb2 Handle pre-generated buildnumber 76b05af Pass the parent build stamp to child jobs; Use the same Atlas services source code commit in all child jobs f7cc216 Pass the parent build stamp to child jobs; Use the same Atlas services source code commit in all child jobs 51156e2 Merge commit '447b5234db39500e985b7900bc2264319f4b5ae3' into master 815a9e9 Remove ACM from Vertical stack (as it is using different versions of Atlas); Import the MCS version that was used to build Atlas UI 793c979 Remove ACM from Vertical stack (as it is using different versions of Atlas); Import the MCS version that was used to build Atlas UI a1340ea Provide specific parameter names appropriate to type of post build job being called 94ba8c7 Provide specific parameter names appropriate to type of post build job being called df17c09 Correct import of print server binaries 4f57c1c Remove some post build log 724179c Use extended email to send emails on failed build c4ea96a Use extended email to send emails on failed build 414a85f Use extended email to send emails on failed build eb7eed4 Use extended email to send emails on failed build ae2c515 Use extended email to send emails on failed build 39fcf5e Use extended email to send emails on failed build 804929b Use extended email to send emails on failed build 752ed54 Use extended email to send emails on failed build b2a9f19 Use extended email to send emails on failed build 7b58c8c Use extended email to send emails on failed build 944b95b Use extended email to send emails on failed build 941d5b3 Use extended email to send emails on failed build 5053f37 Use extended email to send emails on failed build 6779041 Use extended email to send emails on failed build 90e8e67 Use extended email to send emails on failed build f93acc5 Use extended email to send emails on failed build 404cdef Use extended email to send emails on failed build 66ffedf Use extended email to send emails on failed build f41825d Use extended email to send emails on failed build 1bf41f1 Use extended email to send emails on failed build 7cfd3f2 Use extended email to send emails on failed build 0469b40 Use extended email to send emails on failed build ece760f Use extended email to send emails on failed build efa2900 Use extended email to send emails on failed build d7fae34 Use extended email to send emails on failed build c681874 Add failed build action 33208a4 Add failed build action 61a519d Add failed build action f7b0a22 Add failed build action bb477a2 Add failed build action 1ad79a3 Add failed build action ee1ad67 Add failed build action 0ec7308 Add failed build action 3b9ba8b Add failed build action 1a67b16 Add failed build action 12c6b71 Add failed build action 1b4e670 Add failed build action 8b0c194 Add failed build action 607232d Add failed build action 3f5001a Add use of post build info json 0aeadaa Add use of post build info json 0486d41 Add use of post build info json dc894e9 Add use of post build info json ec8ad90 Add use of post build info json a5b06d7 Add use of post build info json 951d694 Add use of post build info json fa33404 Add use of post build info json 458d960 Add use of post build info json e90a217 Add use of post build info json 2cbe8f3 Add use of post build info json 535076b Add use of post build info json ba2fa8c Add a JSON object passing mechanism 2f65f29 Add a JSON object passing mechanism 037f00f Add a JSON object passing mechanism ee23488 Check for more than one commit associated with source git urls 3d2a103 Check for product components using the same release git url c8373c6 Check for product components using the same release git url 92e165b Check for product components using the same release git url b04f5b0 Add a diff post build action 2d3aa7f Resurrect ccm 5d8860f Resurrect ccm ca5436b Resurrect ccm 0ac129b Resurrect ccm a86ca86 Update parameter usage on git checkout c87c3bb Add a diff post build action 7db1c3a Add a diff post build action fe32259 Handle different types of build selector parameter ec9f9e9 Handle different types of build selector parameter 5ee449e Handle different types of build selector parameter 6a8d736 Use the version of print server that atlas services is using - so that there are less build configuration issues 68422e3 Use the version of print server that atlas services is using - so that there are less build configuration issues 458a2e2 Use the version of print server that atlas services is using - so that there are less build configuration issues 904f3c4 Use the version of print server that atlas services is using - so that there are less build configuration issues ec0b7d7 Use the version of print server that atlas services is using - so that there are less build configuration issues 5e71ed6 Merge commit '9177787fd425d22f7ea0797af127b011b8b66094' into core-502_VerticalStackRefactor git-subtree-dir: autobuild/Common_scripts git-subtree-split: 6f0a4482be9cd4dec08d07b9e58dab1611fb5123 --- artifacts/archiver.groovy | 17 + artifacts/metaexport.groovy | 7 +- build/integrity.groovy | 74 ++++- build/postactions.groovy | 258 ++++++++++++--- build/postactionstest.groovy | 43 +++ check/build_meta.groovy | 3 +- check/do_meta.groovy | 6 +- configuration/configuration.groovy | 4 +- copy/all.groovy | 33 +- copy/esw.groovy | 8 +- copy/filtered.groovy | 7 +- copy/logic.groovy | 40 ++- differences/dometahierarchy.groovy | 12 +- differences/metahierarchy.groovy | 17 +- hardware/sominfo.groovy | 80 +++++ import/importer.groovy | 51 ++- meta/folder.groovy | 506 ++++++++++++++++++++++------- packaging/debian.groovy | 64 ++-- publish/do.groovy | 12 +- publish/package.groovy | 7 +- publish/promptingpublish.groovy | 6 +- publish/publish.groovy | 3 +- publish/publishitems.groovy | 10 +- release/do.groovy | 40 ++- release/release.groovy | 27 +- staticanalysis/ccm.groovy | 58 +--- staticanalysis/cpd.groovy | 3 +- staticanalysis/cppcheck.groovy | 3 +- test/testethernetproduct.groovy | 3 +- versioncontrol/git.groovy | 6 +- versionnumbering/generate.groovy | 10 + 31 files changed, 1057 insertions(+), 361 deletions(-) create mode 100644 artifacts/archiver.groovy create mode 100644 build/postactionstest.groovy create mode 100644 hardware/sominfo.groovy diff --git a/artifacts/archiver.groovy b/artifacts/archiver.groovy new file mode 100644 index 0000000..bafba02 --- /dev/null +++ b/artifacts/archiver.groovy @@ -0,0 +1,17 @@ +enum Store { + JENKINS, + ARTIFACTSERVER, +} + +this.JENKINS = Store.JENKINS +this.ARTIFACTSERVER = Store.JENKINS + +def archive(MetaInfo meta, String artifactlist, Store store = JENKINS) { + echo "Archiving artifacts to store: ${store}" + meta.copyrelatedmeta() + if (store == JENKINS) { + archiveArtifacts(artifacts: artifactlist) + } +} + +return this \ No newline at end of file diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 4d93cc5..8e5e56e 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -11,7 +11,12 @@ def combineSubjobArtifacts(meta, importer, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(subJobName, subJobNumber) + importoptions.setincrementversionnumber(incrementversionnumber) + importer.artifactimport(meta, importoptions, destFolder) + artifactFolders.add("${destFolder}/**") } } diff --git a/build/integrity.groovy b/build/integrity.groovy index 73d5d46..e93c00f 100644 --- a/build/integrity.groovy +++ b/build/integrity.groovy @@ -10,6 +10,7 @@ def check(buildmeta) { def allmeta = buildmeta.allrelatedmeta() echo "All meta:\n${allmeta.comment}" + // Store build stamps associated with each product component; Store a counter against each product component version def productcomponents = [:] for (meta in allmeta.metacollection) { def key = meta.componentidentifier() @@ -24,9 +25,13 @@ def check(buildmeta) { productcomponents[key][buildstampnumber] += 1 } } + + // Show what was collected echo prettyPrint(toJson(productcomponents)) + + // Look for product components with more than one build stamp for (pc in productcomponents.keySet()) { - bsns = productcomponents[pc] + def bsns = productcomponents[pc] if (bsns.size() > 1) { results.good = false results.comment += "There is more than one version of Product component: ${pc} being used in this build:\n" @@ -35,6 +40,73 @@ def check(buildmeta) { } } } + + // Store product components associated with each release git url + def releasegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.releaseinfo.releaserepoURL + + if (key) { + if (!releasegiturls.containsKey(key)) { + releasegiturls[key] = [:] + } + def componentidentifier = meta.componentidentifier() + if (!releasegiturls[key].containsKey(componentidentifier)) { + releasegiturls[key][componentidentifier] = 1 + } else { + releasegiturls[key][componentidentifier] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(releasegiturls)) + + // Look for release git urls being used by more than one product component + for (rgu in releasegiturls.keySet()) { + def cis = releasegiturls[rgu] + if (cis.size() > 1) { + results.good = false + results.comment += "There is more than one product component associated with a release git url: ${rgu} being used in this build:\n" + for (ci in cis.keySet()) { + results.comment += "${ci}\n" + } + } + } + + // Store git commit shas associated with each source git url + def sourcegiturls = [:] + for (meta in allmeta.metacollection) { + def String key = meta.gitinfo.remoteurl + + if (key) { + if (!sourcegiturls.containsKey(key)) { + sourcegiturls[key] = [:] + } + def commitsha = meta.gitinfo.gitrevision + if (!sourcegiturls[key].containsKey(commitsha)) { + sourcegiturls[key][commitsha] = 1 + } else { + sourcegiturls[key][commitsha] += 1 + } + } + } + + // Show what was collected + echo prettyPrint(toJson(sourcegiturls)) + + // Look for source git urls associated with more than one sha code + for (sgu in sourcegiturls.keySet()) { + def cs = sourcegiturls[sgu] + if (cs.size() > 1) { + results.good = false + results.comment += "There is more than one source commit sha associated with a source git url: ${sgu} being used in this build:\n" + for (c in cs.keySet()) { + results.comment += "${c}\n" + } + } + } + return results } diff --git a/build/postactions.groovy b/build/postactions.groovy index 48ca297..8b18908 100644 --- a/build/postactions.groovy +++ b/build/postactions.groovy @@ -1,63 +1,223 @@ +/* Example post build info +{ + "Build outcomes": { + "Success": { + "Actions": { + "Deploy documentation": { + "Description": "Can be empty - Name of documentation/doxygen deploy job to run if this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_doxygen_deploy" + }, + "Static analysis": { + "Description": "Can be empty - Static analysis job to run when this build is successful", + "Job": "" + }, + "Unit test": { + "Description": "Can be empty - Unit test job to run when this build is successful", + "Job": "" + }, + "Regression test": { + "Description": "Can be empty - Regression test job to run when this build is successful", + "Job": "HMB_FD_HV_0.9.6/HMB_FD_HV_0.9.6_sg1024_run_tests" + }, + "Update differences": { + "Description": "Can be empty - Update differences job to run when this build is successful", + "Job": "HMB_FD_HV_diff_releases" + }, + "Release to git": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "Job": "HMB_FD_HV_release_to_git" + } + } + }, + "Failed": { + "Actions": { + "Email": { + "Description": "Can be empty - Name of release to git job to run if this build is successful", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ] + }, + "ExtEmail": { + "Description": "Send an email using Extended Email plugin", + "To": [ + "dan.hawkins@globalinkjetsystems.com" + ], + "Recipient providers": [ + "culprits", + "requestor", + "developers", + "upstreamDevelopers" + ] + } + } + } + } +} +*/ + +def getActionJobName(postbuildactions, String action, String alternativeparamname) { + def String jobname = '' + if (postbuildactions) { + try { + jobname = postbuildactions[action]['Job'] + } catch(Exception ex) { + echo "No post build actions found for action: ${action}" + jobname = '' + } + } else { + if (params[alternativeparamname] != null) { + if (params[alternativeparamname]) { + jobname = params[alternativeparamname] + } + } + } + if (jobname) { + def jobitem = Jenkins.instance.getItemByFullName(jobname) + if (jobitem) { + if (!jobitem.isBuildable()) { + jobname = '' + } + } else { + jobname = '' + } + } + // echo "Job name for action: ${action}: ${jobname}" + return jobname +} + +def deployjob(String buildselector, postbuildactions, String action, String alternativeparamname, String jobnameparametername, String jobselectorparametername) { + def jobname = getActionJobName(postbuildactions, action, alternativeparamname) + if (jobname) { + build( + wait: false, + job: jobname, + parameters: [ + string(name: jobnameparametername, value: JOB_NAME), + string(name: jobselectorparametername, value: buildselector), + ] + ) + } +} + +def readPostBuildInfo() { + def postbuildinfo = readJSON(text: POST_BUILD_INFO) + def postbuildjsontext = writeJSON(returnText: true, json: postbuildinfo, pretty: 1) + // echo "Post build info:\n${postbuildjsontext}" + + return postbuildinfo +} def performSuccessfulBuildActions() { - if (params.DOXYGEN_DEPLOY_JOB != null) { - if (DOXYGEN_DEPLOY_JOB && Jenkins.instance.getItemByFullName(DOXYGEN_DEPLOY_JOB).isBuildable()) { - build( - wait: false, - job: DOXYGEN_DEPLOY_JOB, - parameters: [ - string(name: 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', value: JOB_NAME), - string(name: 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR', value: " ${BUILD_NUMBER}"), - ] - ) + def postbuildactions + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Success']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Success/Actions" + postbuildactions = null } } - if (params.STATIC_ANALYSIS_JOB != null) { - if (STATIC_ANALYSIS_JOB && Jenkins.instance.getItemByFullName(STATIC_ANALYSIS_JOB).isBuildable()) { - build( - wait: false, - job: STATIC_ANALYSIS_JOB, - parameters: [ - string(name: 'APPLICATION_ARTIFACT_PROJECT', value: JOB_NAME), - string(name: 'APPLICATION_ARTIFACT_SELECTOR', value: " ${BUILD_NUMBER}"), - ] + def String buildselector = "${BUILD_NUMBER}" + deployjob(buildselector, postbuildactions, 'Deploy documentation', 'DOXYGEN_DEPLOY_JOB', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS', 'BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR') + deployjob(buildselector, postbuildactions, 'Static analysis', 'STATIC_ANALYSIS_JOB', 'BUILD_JOB_TO_ANALYSE', 'BUILD_JOB_TO_ANALYSE_SELECTOR' ) + deployjob(buildselector, postbuildactions, 'Unit test', 'UNIT_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Regression test', 'REGRESSION_TEST_JOB', 'BUILD_JOB_TO_TEST', 'BUILD_JOB_TO_TEST_SELECTOR') + deployjob(buildselector, postbuildactions, 'Update differences', 'DIFF_JOB', 'LATER_BUILD_JOB', 'LATER_BUILD_SELECTOR') + deployjob(buildselector, postbuildactions, 'Release to git', 'RELEASE_TO_GIT_JOB', 'BUILD_JOB', 'BUILD_JOB_SELECTOR') +} + +def sendemails(emailinfo) { + def emailtolist = emailinfo['To'] + + if (emailtolist) { + def emailbody = """ + Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME} : ${BUILD_URL} + + Console output: ${BUILD_URL}console + """ + for(emailto in emailtolist) { + mail( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: emailbody, + to: emailto ) + // echo "Send email to: ${emailto}" } } - if (params.UNIT_TEST_JOB != null) { - if (UNIT_TEST_JOB && Jenkins.instance.getItemByFullName(UNIT_TEST_JOB).isBuildable()) { - build( - wait: false, - job: UNIT_TEST_JOB, - parameters: [ - string(name: 'APPLICATION_ARTIFACT_PROJECT', value: JOB_NAME), - string(name: 'APPLICATION_ARTIFACT_SELECTOR', value: " ${BUILD_NUMBER}"), - ] - ) +} + +def getRecipientProviders(recipientProviderInfoList) { + def recipientProviderMap = [ + culprits: [$class: 'CulpritsRecipientProvider'], + requestor: [$class: 'RequesterRecipientProvider'], + developers: [$class: 'DevelopersRecipientProvider'], + upstreamDevelopers: [$class: 'UpstreamComitterRecipientProvider'], + ] + def recipientProviders = [] + + if (recipientProviderInfoList) { + for(rp in recipientProviderInfoList) { + if (recipientProviderMap.containsKey(rp)) { + recipientProviders.add(recipientProviderMap[rp]) + } } } - if (params.REGRESSION_TEST_JOB != null) { - if (REGRESSION_TEST_JOB && Jenkins.instance.getItemByFullName(REGRESSION_TEST_JOB).isBuildable()) { - build( - wait: false, - job: REGRESSION_TEST_JOB, - parameters: [ - string(name: 'APPLICATION_ARTIFACT_PROJECT', value: JOB_NAME), - string(name: 'APPLICATION_ARTIFACT_SELECTOR', value: " ${BUILD_NUMBER}"), - ] - ) + if (recipientProviders.size() == 0) { + recipientProviders = [culprits(), requestor(), developers(), upstreamDevelopers()] + } + // echo "recipientProviders: ${recipientProviders.toString()}" + return recipientProviders; +} + +def String getToList(toListInfo) { + def String toList = '' + + if (toListInfo) { + toList = toListInfo.join(',') + } + // echo "toList: ${toList}" + return toList +} + +def sendextemail(extmailinfo) { + def recipientProviders = getRecipientProviders(extmailinfo['Recipient providers']) + def toList = getToList(extmailinfo['To']) + + emailext( + subject: "[Jenkins] Failed build: ${JOB_NAME} ${BUILD_DISPLAY_NAME}", + body: '''${JELLY_SCRIPT,template="html"}''', + attachLog: true, + // attachmentsPattern: '', + compressLog: false, + // from : '', + mimeType: 'text/html', + // postsendScript: '', + // presendScript: '', + recipientProviders: recipientProviders, // Could be: buildUser, culprits, developers, brokenTestsSuspects, brokenBuildSuspects, recipients, requestor, upstreamDevelopers + // replyTo: '', + saveOutput: false, + to: toList, + ) +} + +def performFailedBuildActions() { + def postbuildactions = null + if (params.POST_BUILD_INFO != null) { + def postbuildinfo = readPostBuildInfo() + try { + postbuildactions = postbuildinfo['Build outcomes']['Failed']['Actions'] + } catch(Exception ex) { + echo "Post build info did not contain: Build outcomes/Failed/Actions" + postbuildactions = null } } - if (params.RELEASE_TO_GIT_JOB != null) { - if (RELEASE_TO_GIT_JOB && Jenkins.instance.getItemByFullName(RELEASE_TO_GIT_JOB).isBuildable()) { - build( - wait: false, - job: RELEASE_TO_GIT_JOB, - parameters: [ - string(name: 'BUILD_JOB', value: JOB_NAME), - string(name: 'BUILD_JOB_SELECTOR', value: " ${BUILD_NUMBER}"), - ] - ) + if (postbuildactions) { + if (postbuildactions['Email']) { + sendemails(postbuildactions['Email']) + } + if (postbuildactions['ExtEmail']) { + sendextemail(postbuildactions['ExtEmail']) } } } diff --git a/build/postactionstest.groovy b/build/postactionstest.groovy new file mode 100644 index 0000000..150f5d1 --- /dev/null +++ b/build/postactionstest.groovy @@ -0,0 +1,43 @@ +pipeline { + agent { label 'master' } + stages { + stage('Initial setup') { + steps { + script { + CLEAN_WORKSPACE = false + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Dummy build job') { + steps { + script { + echo "Execute dummy build job: ${CAUSE_ERROR}" + if (CAUSE_ERROR == 'true') { + error("Causing error") + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performFailedBuildActions() + } + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + script { + load("${autobuilder.commonscripts()}/build/postactions.groovy").performSuccessfulBuildActions() + } + } + } +} diff --git a/check/build_meta.groovy b/check/build_meta.groovy index 6e669bc..fb8a75d 100644 --- a/check/build_meta.groovy +++ b/check/build_meta.groovy @@ -53,7 +53,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 7bfe2d1..4b8a89e 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = entry.meta.componentversionuniqueidentifier() - def offsetpath = entry.offsetpath + def name = entry.metainfo.componentversionuniqueidentifier() + def offsetpath = entry.accumulatedpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } @@ -65,7 +65,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo2 = componentReleaseInfo[key] - def offsetpath = buildinfo2.offsetpath + def offsetpath = buildinfo2.accumulatedpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 2a7f1ab..7a89975 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,7 +1,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) + return meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) } def registerPipelineScriptConfiguration(meta) { @@ -12,7 +12,7 @@ def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, St def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) + return registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this diff --git a/copy/all.groovy b/copy/all.groovy index 6d09903..aab479c 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -5,6 +5,8 @@ pipeline { steps { script { meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") } } } @@ -23,7 +25,6 @@ pipeline { stage('Import source artifacts') { steps { script { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def copyoptions = importer.getNewImportOptions() copyoptions.source( SOURCE_ARTIFACT_LOCATION, @@ -37,9 +38,14 @@ pipeline { } } } - stage('Copy source artifacts') { + stage('Archiving artifacts') { steps { script { + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + dir(IMPORTS) { fileOperations([ fileCopyOperation(excludes: '', @@ -48,23 +54,10 @@ pipeline { targetLocation: WORKSPACE) ]) } - } - } - } - stage('Archiving artifacts') { - steps { - script { - def includes = '**/*' - def excludes = "${IMPORTS}/*" - echo "Archiving artifacts: Includes: ${includes} Excludes: ${excludes}" - // This does not work: archiveArtifacts artifacts: includes, excludes: excludes - // so - cleanfolders = [ - IMPORTS, - 'autobuild', - ] - cleaner.clean(cleanfolders) - archiveArtifacts artifacts: includes + + cleaner.clean([IMPORTS]) + + artifactarchiver.archive(meta, artifactlist) } } } @@ -83,4 +76,4 @@ pipeline { echo 'Succeeded build' } } -} \ No newline at end of file +} diff --git a/copy/esw.groovy b/copy/esw.groovy index c6c0147..b9eb737 100644 --- a/copy/esw.groovy +++ b/copy/esw.groovy @@ -28,7 +28,10 @@ pipeline { stage('Copying embedded software artifacts') { steps { script { - importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST) + def eswimportoptions = importer.getNewImportOptions() + eswimportoptions.source(ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION) + eswimportoptions.setincrementversionnumber(true) + importer.artifactimport(meta, eswimportoptions, ESWDEST) } } } @@ -41,7 +44,8 @@ pipeline { rm -v -rf meta mv -v "${ESW}"/* . """ - archiveArtifacts artifactlist + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/copy/filtered.groovy b/copy/filtered.groovy index 7fd87b6..3ad24cd 100644 --- a/copy/filtered.groovy +++ b/copy/filtered.groovy @@ -27,7 +27,9 @@ pipeline { stage('Import artifacts') { steps { script { - importer.artifactimport(meta, ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION, IMPORT) + def importoptions = importer.getNewImportOptions() + importoptions.source(ARTIFACT_LOCATION, ARTIFACT_PROJECT, ARTIFACT_BUILD_SELECTOR, RELEASE_REPO, RELEASE_REVISION) + importer.artifactimport(meta, importoptions, IMPORT) } } } @@ -48,7 +50,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["filtered/**"]) - archiveArtifacts artifactlist + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/copy/logic.groovy b/copy/logic.groovy index 9585f01..db9dbc3 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -13,22 +13,27 @@ pipeline { stage('Clean exports') { steps { script { - env.LOGICDEST = "${WORKSPACE}/logic" + LOGICDEST = 'logic' cleanfolders = [ LOGICDEST, "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/el", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Copying logic artifacts') { + stage('Import source artifacts') { steps { script { - sh('env') def logicoptions = importer.getNewImportOptions() - logicoptions.source(LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_ARTIFACT_RELEASE_REPO, LOGIC_ARTIFACT_RELEASE_REVISION) + logicoptions.source( + LOGIC_ARTIFACT_LOCATION, + LOGIC_ARTIFACT_PROJECT, + LOGIC_ARTIFACT_BUILD_SELECTOR, + LOGIC_ARTIFACT_RELEASE_REPO, + LOGIC_ARTIFACT_RELEASE_REVISION) importer.artifactimport(meta, logicoptions, LOGICDEST) } } @@ -36,14 +41,23 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["embedded_logic/**"]) - sh """ - rm -v -rf meta - # The following is removing a deprecated folder - rm -v -rf diff - mv -v "logic"/* . - """ - archiveArtifacts artifactlist + def artifactlist = meta.generateartifacts(['**/*']) + def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + + cleaner.clean(['meta', 'autobuild']) + + dir(LOGICDEST) { + fileOperations([ + fileCopyOperation(excludes: '', + flattenFiles: false, + includes: '**/*', + targetLocation: WORKSPACE) + ]) + } + + cleaner.clean([LOGICDEST]) + + artifactarchiver.archive(meta, artifactlist) } } } @@ -62,4 +76,4 @@ pipeline { echo 'Succeeded build' } } -} \ No newline at end of file +} diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 6280960..3b25798 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -95,8 +95,8 @@ def showmetaitems(metaitems) { String info = '' for (key in metaitems.keySet()) { def relatedmetainfo = metaitems[key] - def offsetpath = relatedmetainfo.offsetpath - info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" + def accumulatedpath = relatedmetainfo.accumulatedpath + info += "Name: ${key}\t\t\t\tPath: ${accumulatedpath}\n" } echo info } @@ -768,8 +768,8 @@ def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" - def earlymeta = earlymetaitems[key].meta - def latermeta = latermetaitems[key].meta + def earlymeta = earlymetaitems[key].metainfo + def latermeta = latermetaitems[key].metainfo summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") @@ -803,8 +803,8 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin def earlymeta = meta.importrelated(true, earlymetafolder) def latermeta = meta.importrelated(true, latermetafolder) - def earlymetaitems = earlymeta.releasewalkerunique(md) - def latermetaitems = latermeta.releasewalkerunique(md) + def earlymetaitems = earlymeta.releasewalkerunique(md, '', earlymetafolder) + def latermetaitems = latermeta.releasewalkerunique(md, '', latermetafolder) echo "Early items:" showmetaitems(earlymetaitems) diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index 10b24a0..01c1890 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -37,8 +37,18 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') - importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') + + def earlyimportoptions = importer.getNewImportOptions() + earlyimportoptions.source(EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION) + earlyimportoptions.setincrementversionnumber(false) + earlyimportoptions.setfilter('meta/**') + importer.artifactimport(meta, earlyimportoptions, EARLYROOTDIR) + + def laterimportoptions = importer.getNewImportOptions() + laterimportoptions.source(LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION) + laterimportoptions.setincrementversionnumber(false) + laterimportoptions.setfilter('meta/**') + importer.artifactimport(meta, laterimportoptions, LATERROOTDIR) } } } @@ -58,7 +68,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,reports/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,reports/**,${artifactlist}") } } } diff --git a/hardware/sominfo.groovy b/hardware/sominfo.groovy new file mode 100644 index 0000000..72a3ca9 --- /dev/null +++ b/hardware/sominfo.groovy @@ -0,0 +1,80 @@ +def sominfostore = null + +/* Example expected JSON +{ + "SomType": { + "Som_BA4": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_ba4", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_ba4.git", + "Release repo revision": "" + } + }, + "Som_FC5": { + "Artifact info": { + "Location": "Jenkins job artifact", + "Job": "HMB_FD_HV_0.9.6/0.9.6_SG1024_EL_syn_fc5", + "Job selector": "true", + "Release repo": "git@hydra:hmb_fd_hv-releases/sg1024/hmb_fd_hv_logic_fc5.git", + "Release repo revision": "" + } + } + } +} +*/ + +def String asJsonText() { + return writeJSON(returnText: true, json: sominfostore, pretty: 1) +} + +def boolean exists() { + return (sominfostore != null) +} + +def set(String jsontext) { + sominfostore = readJSON(text: jsontext) + echo "Som support info:\n${asJsonText()}" +} + +def get() { + return sominfostore +} + +def somtypes() { + return sominfostore['SomType'] +} + +def artifactinfo(somtype) { + return somtype['Artifact info'] +} + +def artifactlocation(artifactinfo) { + return artifactinfo['Location'] +} + +def boolean isjoblocation(artifactinfo) { + return (artifactinfo['Location'] == 'Jenkins job artifact' || + artifactinfo['Location'] == 'Jenkins job artefact') +} + +def boolean isreleaserepolocation(artifactinfo) { + return artifactinfo['Location'] == 'Release repository' +} + +def artifactjob(artifactinfo) { + return [ + "name": artifactinfo['Job'], + "selector": "${artifactinfo['Job selector']}" + ] +} + +def releaserepo(artifactinfo) { + return [ + "repo": artifactinfo['Release repo'], + "revision": artifactinfo['Release repo revision'] + ] +} + +return this \ No newline at end of file diff --git a/import/importer.groovy b/import/importer.groovy index 8624421..8bb565a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -96,9 +96,21 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void jenkinsjob(String jobname, String buildnumberorselector) { + void jenkinsjob(String jobname, buildnumberorselector) { m_jobname = jobname.trim() - m_buildnumberorselector = buildnumberorselector.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: buildnumberorselector + ] + setsourceselect('Jenkins job artifact') + } + + void jenkinsjob(String jobname, int buildnumberorselector) { + m_jobname = jobname.trim() + m_buildnumberorselector = [ + $class: 'ParameterizedBuildSelector', + parameterName: "${buildnumberorselector}" + ] setsourceselect('Jenkins job artifact') } @@ -147,15 +159,14 @@ def artifactimportfrombuildjob(meta, importoptions, String destpath) { conditionalClean(importoptions, destpath) + echo "Build selector type: ${importoptions.m_buildnumberorselector.getClass()}" + echo "Importing artifacts from job: ${importoptions.m_jobname} selector: ${importoptions.m_buildnumberorselector.toString()}" copyArtifacts( filter: formFilter(importoptions), excludes: importoptions.m_excludes, fingerprintArtifacts: importoptions.m_fingerprint, projectName: importoptions.m_jobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: importoptions.m_buildnumberorselector - ], + selector: importoptions.m_buildnumberorselector, flatten: importoptions.m_flatten, target: destpath ) @@ -276,7 +287,7 @@ def artifactimport(meta, boolean fingerprint = false, boolean nested = false) { echo "DEPRECATED method: artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.source(sourceselect, jenkinsjob, buildnumberorselector, repo, revision) importoptions.setincrementversionnumber(incrementversionnumber) @@ -298,7 +309,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false) { echo "DEPRECATED method: artifactimport(meta, String buildjob, int buildnumber, String destpath, ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(buildjob, " ${buildnumber}") importoptions.setincrementversionnumber(incrementversionnumber) @@ -319,7 +330,7 @@ def artifactimport(meta, String excludes = '', boolean fingerprint = false) { echo "DEPRECATED method: artifactimport(meta, String buildjob, buildnumberorselector, String destpath, ...)" - def importoptions = getNewImportOptions() + def importoptions = importer.getNewImportOptions() importoptions.jenkinsjob(buildjob, buildnumberorselector) importoptions.setincrementversionnumber(incrementversionnumber) @@ -330,4 +341,26 @@ def artifactimport(meta, return artifactimport(meta, importoptions, destpath) } +// Import the artifacts of a named child component found below parent meta's hierarchy +def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, String childsubcomponent, importoptions, String destinationfolder) { + // Import from parent meta + def childmeta = meta.getNewMetaObject() + def childmeta2 + echo "Finding meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}" + echo "From parent meta identifier: ${parentmeta.componentidentifier()} and parent meta path: ${parentmetapath}" + def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) + if (findresults.matchfound) { + childmeta = findresults.matchedmetaitem.metainfo + if (childmeta.releaseinfo.released) { + importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) + } else { + importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) + } + childmeta2 = importer.artifactimport(meta, importoptions, destinationfolder) + } else { + error("Could not find meta associated with child: Product: ${childproduct} Sub-component: ${childsubcomponent}") + } + return childmeta2 +} + return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index aee84f6..b9e898c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -20,15 +20,19 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - String productname - String subcomponentname - String variant + private String format + private String productname + private String subcomponentname + private String variant + private String uniqueid - ProductInfo(script, productname = "", subcomponentname = "", variant = "") { + ProductInfo(script, String productname = '', String subcomponentname = '', String variant = '') { super(script) + this.format = '0.1' this.productname = productname this.subcomponentname = subcomponentname this.variant = variant + this.uniqueid = UUID.randomUUID().toString() } void set(String productname, String subcomponentname, String variant = '') { @@ -74,33 +78,40 @@ class ProductInfo extends GISJsonSerialised { this.productname = otherinfo.productname this.subcomponentname = otherinfo.subcomponentname this.variant = otherinfo.variant + this.uniqueid = otherinfo.uniqueid } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] productname = rawmeta['productname'] subcomponentname = rawmeta['subcomponentname'] - variant = rawmeta['variant'] ?: "" + variant = rawmeta['variant'] ?: '' + uniqueid = rawmeta['uniqueid'] ?: uniqueid } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'productname' : productname, 'subcomponentname' : subcomponentname, 'variant' : variant, + 'uniqueid': uniqueid, ] return data } } class GitInfo extends GISJsonSerialised { + private String format private String sourcepath private String sourcerevision private String gitrevision private String remoteurl private String remoteinfo - GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + GitInfo(script, sourcepath = '', sourcerevision = '', gitrevision = '', remoteurl = '', remoteinfo = '') { super(script) + this.format = '0.1' this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -134,6 +145,7 @@ class GitInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] sourcepath = rawmeta['sourcepath'] sourcerevision = rawmeta['sourcerevision'] gitrevision = rawmeta['gitrevision'] @@ -143,6 +155,7 @@ class GitInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, 'gitrevision' : gitrevision, @@ -188,15 +201,17 @@ class JenkinsBuildParameter extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { - String buildjobname - int buildnumber + private String format + private String buildjobname + private int buildnumber private String buildmachine private String builddate - ArrayList parameters - boolean isfullbuild + private ArrayList parameters + private boolean isfullbuild - JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "", boolean isfullbuild = true) { + JenkinsBuildInfo(script, String buildjobname = '', int buildnumber = 0, String buildmachine = '', String builddate = '', boolean isfullbuild = true) { super(script) + this.format = '0.1' this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine @@ -228,6 +243,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] buildjobname = rawmeta['buildjobname'] buildnumber = rawmeta['buildnumber'] buildmachine = rawmeta['buildmachine'] @@ -253,6 +269,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, 'buildmachine' : buildmachine, @@ -265,12 +282,14 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - String buildstampprefix - int revision - String buildstamp + private String format + private String buildstampprefix + private int revision + private String buildstamp BuildStampInfo(script, buildstampprefix = "0.30000.0.", revision = 0, buildstamp = "0.30000.0.0") { super(script) + this.format = '0.1' this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp @@ -293,6 +312,20 @@ class BuildStampInfo extends GISJsonSerialised { set(other.buildstampprefix, other.revision, other.buildstamp) } + // Override the buildstamp + void override(String buildstamp) { + this.buildstamp = buildstamp + script.echo("Build stamp overridden to: ${this.buildstamp}") + def s = this.buildstamp.split('\\.') + if (s.size() != 4) { + script.error("Overriding build stamp: ${buildstamp} is not of the format: 1.2.3.4") + } + this.buildstampprefix = "${s[0]}.${s[1]}.${s[2]}" + script.echo("Stamp prefix overridden to: ${this.buildstampprefix}") + this.revision = s[3].toInteger() + script.echo("Revision overridden to: ${this.revision}") + } + String buildstampcommasep() { def String buildstampcommasep = buildstamp.replace(".", ",") return buildstampcommasep @@ -319,6 +352,7 @@ class BuildStampInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] @@ -326,6 +360,7 @@ class BuildStampInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, @@ -335,18 +370,20 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - String releaserepoURL - ArrayList releaseitems - boolean released + private String format + private String releaserepoURL + private ArrayList releaseitems + private boolean released - ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { + ReleaseInfo(script, String releaserepoURL = '', boolean released = false) { super(script) + this.format = '0.1' this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() this.released = released } - void set(String releaserepoURL = "") { + void set(String releaserepoURL = '') { this.releaserepoURL = releaserepoURL } @@ -368,6 +405,7 @@ class ReleaseInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] releaserepoURL = rawmeta['releaserepoURL'] releaseitems = new ArrayList() for (rawreleaseitem in rawmeta['releaseitems']) { @@ -378,6 +416,7 @@ class ReleaseInfo extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'releaserepoURL' : releaserepoURL, 'releaseitems' : releaseitemssserialised(), 'released' : released @@ -392,7 +431,7 @@ class ConfigurationInfo extends GISJsonSerialised { private String configurationversion private String configurationSHA - ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + ConfigurationInfo(script, configurationname = '', configurationURL = '', configurationversion = '', configurationSHA = '') { super(script) this.configurationname = configurationname this.configurationURL = configurationURL @@ -422,7 +461,7 @@ class ChecksumPair extends GISJsonSerialised { private String filepath private String sum - ChecksumPair(script, String filepath = "", String sum = "") { + ChecksumPair(script, String filepath = '', String sum = '') { super(script) this.filepath = filepath this.sum = sum @@ -443,11 +482,13 @@ class ChecksumPair extends GISJsonSerialised { } class ChecksumSet extends GISJsonSerialised { + private String format private String artifactpath private ArrayList checksumpairs - ChecksumSet(script, artifactpath = "") { + ChecksumSet(script, artifactpath = '') { super(script) + this.format = '0.1' this.artifactpath = artifactpath checksumpairs = new ArrayList() } @@ -467,6 +508,7 @@ class ChecksumSet extends GISJsonSerialised { } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] artifactpath = rawmeta['artifactpath'] checksumpairs = new ArrayList() for (rawchecksumpair in rawmeta['checksumpairs']) { @@ -478,6 +520,7 @@ class ChecksumSet extends GISJsonSerialised { LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'artifactpath' : artifactpath, 'checksumpairs' : checksumpairsserialised(), ] @@ -525,19 +568,23 @@ class ChecksumInfo extends GISJsonSerialised { } class PublishableInfo extends GISJsonSerialised { + private String format private String filepath - PublishableInfo(script, String filepath = "") { + PublishableInfo(script, String filepath = '') { super(script) + this.format = '0.1' this.filepath = filepath } void deserialiseJson(rawmeta) { + //format = rawmeta['format'] filepath = rawmeta['filepath'] } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'filepath' : filepath, ] return data @@ -545,43 +592,96 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private static String FORMAT = "0.1" - private static String METAFOLDERNAME = "meta" - private static String METAFILENAME = "info.json" + private static String METAFOLDERNAME = 'meta' + private static String METAFILENAME = 'info.json' private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String format = '0.2' - class RelatedMetaInfo extends GISJsonSerialised { - String offsetpath - MetaInfo meta - - RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + abstract class RelatedMetaInfo extends GISJsonSerialised { + String format + + RelatedMetaInfo(script, format) { super(script) + this.format = format + } + + abstract void deserialiseJson(rawmeta) + abstract LinkedHashMap serialisationdata() + abstract String getrelatedid() + } + + class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { + private MetaInfo meta + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_1(script, String offsetpath = '', MetaInfo meta = null, String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.1') this.offsetpath = offsetpath + this.uniqueid = uniqueid this.meta = meta } void deserialiseJson(rawmeta) { offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] ?: uniqueid meta = new MetaInfo(script) meta.deserialiseJson(rawmeta['meta']) } LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid, 'meta' : meta.serialisationdata(), ] return data } + + String getrelatedid() { + return uniqueid + } } - private String format = FORMAT - ProductInfo productinfo - GitInfo gitinfo - JenkinsBuildInfo jenkinsbuildinfo - BuildStampInfo buildstampinfo - ReleaseInfo releaseinfo - ChecksumInfo checksuminfo + class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { + private String importpath + private String offsetpath + private String uniqueid + + RelatedMetaInfo_v0_2(script, String importpath = '', String offsetpath = '', String uniqueid = UUID.randomUUID().toString()) { + super(script, '0.2') + this.importpath = importpath + this.offsetpath = offsetpath + this.uniqueid = uniqueid + } + + void deserialiseJson(rawmeta) { + //format = rawmeta['format'] + offsetpath = rawmeta['offsetpath'] + uniqueid = rawmeta['uniqueid'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'offsetpath' : offsetpath, + 'uniqueid' : uniqueid + ] + return data + } + + String getrelatedid() { + return uniqueid + } + } + + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + private ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo private LinkedHashMap custominfo @@ -631,6 +731,7 @@ class MetaInfo extends GISJsonSerialised { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") configurationinfo.add(ci) + return ci } void addCustomInfo(String customId, String customValue) { @@ -662,8 +763,8 @@ class MetaInfo extends GISJsonSerialised { } void deserialiseJson(rawmeta) { - if (rawmeta['format'] != FORMAT) { - script.error("Unsupported meta format found at ${filepath}") + if (rawmeta['format'].toString() != '0.1' && rawmeta['format'].toString() != '0.2' ) { + script.error("Unsupported meta format found") return } productinfo.deserialiseJson(rawmeta['productinfo']) @@ -684,9 +785,15 @@ class MetaInfo extends GISJsonSerialised { } custominfo = rawmeta['custominfo'] for (rawrelatedmeta in rawmeta['associatedmeta']) { - def rm = new RelatedMetaInfo(script) - rm.deserialiseJson(rawrelatedmeta) - associatedmeta.add(rm) + if (!rawrelatedmeta['format'] || rawrelatedmeta['format'] == '0.1') { + def rm = new RelatedMetaInfo_v0_1(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } else if (rawrelatedmeta['format'] == '0.2') { + def rm = new RelatedMetaInfo_v0_2(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } } } @@ -733,19 +840,32 @@ class MetaInfo extends GISJsonSerialised { def String adjustedpath = relativize(destpath) def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { + script.echo("Deserialising meta at file: ${importedmetafile} (destpath: ${destpath})") mi.deserialiseJson(importedmetafile) + + RelatedMetaInfo rmi = new RelatedMetaInfo_v0_2(script, importedmetafile, adjustedpath, mi.productinfo.uniqueid) + if (appendtoparentmeta) { + associatedmeta.add(rmi) + } } else { script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") - mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) + addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } - - RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) - if (appendtoparentmeta) { - associatedmeta.add(rmi) - } - return mi } + + def copyrelatedmeta() { + script.echo("Copying related input meta data to output meta") + for (relatedmetainfo in associatedmeta) { + def srcpath = "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" + def destpath = "${METAFOLDERNAME}/${relatedmetainfo.offsetpath}" + script.echo("Copying related meta folder: ${srcpath} to meta output folder: ${destpath} recursively") + script.fileOperations([ + script.folderCreateOperation(folderPath: destpath), + script.folderCopyOperation(sourceFolderPath: srcpath, destinationFolderPath: destpath) + ]) + } + } LinkedHashMap serialisationdata() { def data = [ @@ -768,10 +888,10 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here - ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here - String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" - ) { + def String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { def String artifactlist = METAARTIFACTS releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { @@ -784,31 +904,6 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } - - // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java - private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { - if (maxdepth > 0) { - if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) - releasemetacollection.add(rmi) - } - if (pathsofar) { - pathsofar += "/" - } - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + relatedmetainfo.offsetpath - releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) - } - } - return releasemetacollection - } - - ArrayList releasewalker(int maxdepth, String startpath = "") { - ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - return releasemetacollection - } String componentidentifier() { return productinfo.productname + "_" + productinfo.subcomponentname @@ -830,72 +925,239 @@ class MetaInfo extends GISJsonSerialised { return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber } - Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + class CollectedMeta { + private MetaInfo metainfo + private String accumulatedpath + private String accumulatedmetapath + + CollectedMeta(MetaInfo metainfo, String accumulatedpath = '', String accumulatedmetapath = '') { + this.metainfo = metainfo + this.accumulatedpath = accumulatedpath + this.accumulatedmetapath = accumulatedmetapath + } + } + + class IterationInfo { + private ArrayList metacollection + private Map metamap + private String comment + private int level + private String gatheredpath + private String gatheredmetapath + private boolean limiteddepth + private int maxdepth + + IterationInfo(ArrayList metacollection, + Map metamap, + String comment = '', + int level = 0, + String gatheredpath = '', + String gatheredmetapath = '', + boolean limiteddepth = false, + int maxdepth = 0) { + this.metacollection = metacollection + this.metamap = metamap + this.comment = comment + this.level = level + this.gatheredpath = gatheredpath + this.gatheredmetapath = gatheredmetapath + this.limiteddepth = limiteddepth + this.maxdepth = maxdepth + } + + String show() { + return "Iteration:\nLevel: ${level}\nGathered path: ${gatheredpath}\nGathered meta path: ${gatheredmetapath}\nLimited depth: ${limiteddepth}\nMax depth: ${maxdepth}" + } + + void add(CollectedMeta item) { + metacollection.add(item) + metamap.put(item.metainfo.productinfo.uniqueid, item) + } + } + + def openchildmeta(IterationInfo ii, RelatedMetaInfo relatedmetainfo) { + def MetaInfo mi = new MetaInfo(script) + + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? "${relatedmetainfo.offsetpath}/${METAFOLDERNAME}" : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def importedmetafile = "${nextgatheredmetapath}/${METAFILENAME}" + def boolean imported + // If this has already been read in from a file + if (relatedmetainfo.getrelatedid() in ii.metamap) { + //script.echo("Already read in meta associated with uniqueid: ${relatedmetainfo.getrelatedid()} - Using it") + mi = ii.metamap[relatedmetainfo.getrelatedid()].metainfo + imported = true + } else if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + imported = true + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} - Does not exist") + imported = false + } + + def childmetainfo = [ + 'imported': imported, + 'nextgatheredpath': nextgatheredpath, + 'nextgatheredmetapath': nextgatheredmetapath, + 'childmeta': mi + ] + return childmetainfo + } + + def walkrelated_v0_1(IterationInfo ii) { + //script.echo("Walk related v0.1: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } else { + script.error("There is non v0.1 format related meta info under v0.1 related meta info") + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + def walkrelated(IterationInfo ii) { + //script.echo("Walk related: ${ii.show()}") + if (!ii.limiteddepth || ii.maxdepth > 0) { + ii.add(new CollectedMeta(this, ii.gatheredpath, ii.gatheredmetapath)) + ii.comment += "${ii.level} ".padRight(ii.level * 4) + globalreleasetag() + '\n' + for (relatedmetainfo in associatedmeta) { + //script.echo("Related meta info: ${relatedmetainfo.offsetpath} format: ${relatedmetainfo.format}") + if (relatedmetainfo.format == '0.2') { + def childmetainfo = openchildmeta(ii, relatedmetainfo) + if (childmetainfo.imported) { + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + childmetainfo.nextgatheredpath, + childmetainfo.nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = childmetainfo.childmeta.walkrelated(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } else if (!relatedmetainfo.format || relatedmetainfo.format == '0.1') { + def MetaInfo othermeta = relatedmetainfo.meta + def String nextgatheredpath = (ii.gatheredpath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredpath}/${relatedmetainfo.offsetpath}") + def String nextgatheredmetapath = (ii.gatheredmetapath == '' ? relatedmetainfo.offsetpath : "${ii.gatheredmetapath}/${relatedmetainfo.offsetpath}") + def nextii = new IterationInfo(ii.metacollection, + ii.metamap, + ii.comment, + ii.level + 1, + nextgatheredpath, + nextgatheredmetapath, + ii.limiteddepth, + ii.maxdepth - 1) + def results = othermeta.walkrelated_v0_1(nextii) + ii.metacollection = results.metacollection + ii.metamap = results.metamap + ii.comment = results.comment + } + } + } + return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] + } + + def allrelatedmeta(String metapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + return walkrelated(new IterationInfo(metacollection, metamap, '', 0, '', metapath)) + } + + ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = new ArrayList() + def Map metamap = new LinkedHashMap() + def results = walkrelated(new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth)) + return results.metacollection + } + + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { - releasemetacollection.put(uniquestring, relatedmetainfo) + def existingentrypathlength = existingentry.accumulatedpath.split('/').size() + def newentrypathlength = metaitem.accumulatedpath.split('/').size() + if (existingentrypathlength > newentrypathlength) { + releasemetacollection.put(uniquestring, metaitem) } } else { - releasemetacollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, metaitem) } return releasemetacollection } - - Map releasewalkerunique(int maxdepth, String startpath = "") { - def ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) - def Map releasemetauniquecollection = new LinkedHashMap() - for (relatedmetainfo in releasemetacollection) { - def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) + Map releasewalkerversionunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") + + def Map versionuniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentversionuniqueidentifier() + versionuniquecollection = putwithshortestoffsetpath(versionuniquecollection, uniquestring, metaitem) } - return releasemetauniquecollection + return versionuniquecollection } - Map releasewalkerversionunique(int maxdepth, String startpath = "") { - ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + Map releasewalkerunique(int maxdepth, String startpath = '', String startmetapath = '') { + def ArrayList metacollection = releasewalker(maxdepth, startpath, "${startmetapath}/${METAFOLDERNAME}") - def Map releasemetaversionuniquecollection = new LinkedHashMap() - for (relatedmetainfo in releasemetacollection) { - def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) + def Map uniquecollection = new LinkedHashMap() + for (metaitem in metacollection) { + def String uniquestring = metaitem.metainfo.componentidentifier() + uniquecollection = putwithshortestoffsetpath(uniquecollection, uniquestring, metaitem) } - return releasemetaversionuniquecollection - } - - def walkrelated(ArrayList metacollection, String comment = '', int level = 0) { - def results = [ - 'metacollection' : metacollection, - 'comment': comment - ] - results.metacollection.add(this) - results.comment += "${level} ".padRight(level * 4) + globalreleasetag() + '\n' - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - results = othermeta.walkrelated(results.metacollection, results.comment, level + 1) - } - return [ 'metacollection' : results.metacollection, 'comment': results.comment ] - } - - def allrelatedmeta() { - ArrayList metacollection = new ArrayList() - return walkrelated(metacollection) + return uniquecollection } MetaInfo getNewMetaObject(script) { return new MetaInfo(script) } + + def findAssociatedMeta(String parentmetapath, String productname, String subcomponentname) { + def results = [ + 'matchedmetaitem' : new CollectedMeta(this), + 'matchfound' : false + ] + def walkresults = allrelatedmeta("${parentmetapath}/${METAFOLDERNAME}") + for (dependentmetaitem in walkresults.metacollection) { + script.echo("Comparing: ${dependentmetaitem.metainfo.productinfo.productname} ${dependentmetaitem.metainfo.productinfo.subcomponentname}") + if (dependentmetaitem.metainfo.productinfo.productname == productname && + dependentmetaitem.metainfo.productinfo.subcomponentname == subcomponentname) { + script.echo("Found matching component: ${dependentmetaitem.metainfo.globalreleasetag()}") + results.matchedmetaitem = dependentmetaitem + results.matchfound = true + break + } + } + return results + } } MetaInfo metareset(script) { - echo "Resetting meta folder" def mi = new MetaInfo(script) + echo "Resetting meta folder" dir(mi.folder()) { deleteDir() } @@ -903,13 +1165,13 @@ MetaInfo metareset(script) { return mi } -MetaInfo metafromfolder(String folder = "") { +MetaInfo metafromfolder(String folder = '') { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") return buildmeta } -MetaInfo metafromfolderSlurper(String folder = "") { +MetaInfo metafromfolderSlurper(String folder = '') { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") return buildmeta diff --git a/packaging/debian.groovy b/packaging/debian.groovy index c0a4dbb..9f2f4d3 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -64,45 +64,47 @@ def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, S return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp, inpackagepathsuffix) } -def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { - try { - sh """#!/bin/bash - # Exit if any command fails - set -xe +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String credentials_id) { + withCredentials([usernamePassword(credentialsId: credentials_id, + usernameVariable: 'SUDO_USERNAME', + passwordVariable: 'SUDO_PASSWORD')]) { + def sudo_S = "echo ${SUDO_PASSWORD} | sudo -S" + def sudo_Sk = "echo ${SUDO_PASSWORD} | sudo -S -k" + try { + sh """#!/bin/bash + # Exit if any command fails + set -xe - echo "${password}" | sudo -S echo + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + ${sudo_Sk} chown -v -R root: "${packagedefs.DEBPATH}" - sudo chown -v -R root: "${packagedefs.DEBPATH}" + mkdir -v -p "${packageout}" + echo "Building package:" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" - mkdir -v -p "${packageout}" - echo "Building package:" - dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + mkdir -v -p "${uniquepackageout}" + cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" - mkdir -v -p "${uniquepackageout}" - cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" + echo "Showing package:" + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package contents:" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package:" - dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" - echo "Showing package contents:" - dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package lintian output:" + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - echo "Showing package lintian output:" - lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" + """ + } + catch (Exception e) { + echo "Exception occurred" + sh """#!/bin/bash -v + ${sudo_Sk} chown -R ${USER}: "${packagedefs.DEBPATH}" - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" - """ - } - catch (Exception e) { - echo "Exception occurred" - sh """#!/bin/bash -v - echo "${password}" | sudo -S echo - - sudo chown -R ${USER}: "${packagedefs.DEBPATH}" - - exit 1 - """ + exit 1 + """ + } } } diff --git a/publish/do.groovy b/publish/do.groovy index 1b227f4..ea090f0 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -60,7 +60,7 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { summaryappend("Artifacts: ") - def releasemeta = releaseinfo.meta + def releasemeta = releaseinfo.metainfo def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo @@ -113,7 +113,7 @@ def copydirectory(File dirFrom, File dirTo) { } def recallcomponent(String productline, releaseinfo, String softwarerelease, itempathstopublish) { - def releasemeta = releaseinfo.meta + def releasemeta = releaseinfo.metainfo def String uniquefolderforrepo = uniqueabsoluterreleasefolder(productline, releaserepofolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") @@ -140,14 +140,14 @@ def convertitemstopublish(String itemstopublishstring) { // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## -def publishhierarchy(hostedftpsiteinfo, releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() preconditions() def productline = releasemeta.productinfo.productname // Walk the meta information to a max depth, looking for artifacts to publish - def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md, '', releasemetapath) echo "Version unique components:" def String versionuniquecomponent = "" @@ -167,8 +167,8 @@ def publishhierarchy(hostedftpsiteinfo, releasemeta, String itemstopublishstring return [summarystring, markunstable, unstablereason] } -def publishhierarchy(hostedftpsiteinfo, releasemeta, String customer, String softwarerelease, String maxdepth) { - return publishhierarchy(hostedftpsiteinfo, releasemeta, "", customer, softwarerelease, maxdepth) +def publishhierarchy(hostedftpsiteinfo, releasemeta, String releasemetapath, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(hostedftpsiteinfo, releasemeta, releasemetapath, '', customer, softwarerelease, maxdepth) } return this; diff --git a/publish/package.groovy b/publish/package.groovy index 958a614..68c41f3 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -48,7 +48,12 @@ pipeline { steps { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) + + def packageimportoptions = importer.getNewImportOptions() + packageimportoptions.jenkinsjob(PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR) + packageimportoptions.setincrementversionnumber(true) + packagemeta = importer.artifactimport(meta, packageimportoptions, PACKAGEDEST) + PACKAGE_FOLDER = "${PACKAGEDEST}/package" UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" } diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 9d51915..ce6a805 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -114,6 +114,7 @@ pipeline { 'Select software items', 'Software items', releasemeta, + SOURCEDIR, ITEMSDEFAULTLIST, MAX_META_DEPTH, '20', @@ -135,7 +136,7 @@ pipeline { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") def hostedftpsiteinfo = hostedftb.getHostedFTPSiteInfo() - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, selecteditems, customer, release, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(hostedftpsiteinfo, releasemeta, SOURCEDIR, selecteditems, customer, release, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' @@ -147,7 +148,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/publish/publish.groovy b/publish/publish.groovy index 82a32ad..ad33df9 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -58,7 +58,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/publish/publishitems.groovy b/publish/publishitems.groovy index 677970e..9f6c368 100644 --- a/publish/publishitems.groovy +++ b/publish/publishitems.groovy @@ -18,6 +18,7 @@ class PublishableItemsSelection { String name, String description, releasemeta, + String releasemetapath, String preselectedext, String metalevelsstringext, String visibleItemCountExt, @@ -25,7 +26,7 @@ class PublishableItemsSelection { outputSelectionfromPreselect(preselectedmenuitems, preselectedext) def Integer metalevelsext = metalevelsstringext.toInteger() def Integer visibleItemCount = visibleItemCountExt.toInteger() - def metaitems = releasemeta.releasewalkerunique(metalevelsext) + def metaitems = releasemeta.releasewalkerunique(metalevelsext, '', releasemetapath) for (key in metaitems.keySet()) { def releaseinfo = metaitems[key] outputDescription(menuitemdescriptions, releaseinfo) @@ -89,7 +90,7 @@ class PublishableItemsSelection { } def outputDescription(menuitems, releaseinfo) { - def meta = releaseinfo.meta + def meta = releaseinfo.metainfo for (publishableitem in meta.publishableinfo) { menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") @@ -97,7 +98,7 @@ class PublishableItemsSelection { } def outputSelection(menuitems, releaseinfo) { - def meta = releaseinfo.meta + def meta = releaseinfo.metainfo for (publishableitem in meta.publishableinfo) { menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") @@ -113,13 +114,14 @@ PublishableItemsSelection createItemSelector( String name, String description, releasemeta, + String releasemetapath, String preselectedext, String metalevelsstringext, String visibleItemCountExt = '20', String type = 'PT_CHECKBOX') { // Could also be "PT_MULTI_SELECT" PublishableItemsSelection pis = new PublishableItemsSelection(this) - pis.init(name, description, releasemeta, preselectedext, metalevelsstringext, visibleItemCountExt, type) + pis.init(name, description, releasemeta, releasemetapath, preselectedext, metalevelsstringext, visibleItemCountExt, type) return pis } diff --git a/release/do.groovy b/release/do.groovy index e0b9471..4dcb091 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -112,20 +112,20 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") def boolean result = true - for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { + for (artifactfilter in buildinfo.metainfo.releaseinfo.releaseitems) { def filterpath = artifactfilter - if (buildinfo.offsetpath) { - filterpath = buildinfo.offsetpath + '/' + artifactfilter + if (buildinfo.accumulatedpath) { + filterpath = buildinfo.accumulatedpath + '/' + artifactfilter } // Copy relevant artifacts to temp folder - echo "Copying artifacts: ${filterpath} to ${tempfolder}" + echo "Copying artifacts: Filter path: ${filterpath} to ${tempfolder} from job: ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname} #${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" try { copyArtifacts( filter: filterpath, projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" ], target: tempfolder ) @@ -136,9 +136,9 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } if (result) { // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + def originalrootdir = tempfolder + "/" + buildinfo.accumulatedpath dir(originalrootdir) { - sh("mv * ${repofolder}") + sh("mv -v * ${repofolder}") } } summaryappend("${artifactfilter},") @@ -375,7 +375,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { - def buildmeta = buildinfo.meta + def buildmeta = buildinfo.metainfo def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl) def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" @@ -396,13 +396,13 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri repoandtagsappend("\n") } -def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { +def releasehierarchy(String buildfolder, String metafolder, buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { def int md = maxdepth.toInteger() resetfolders() preconditions() // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfo = buildmeta.releasewalkerunique(md) + def componentReleaseInfo = buildmeta.releasewalkerunique(md, buildfolder, metafolder) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { @@ -411,14 +411,19 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String echo versioncomponent // Walk the meta information to a max depth, looking for components to release - def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) - echo "Version unique components:" - def String versionuniquecomponent = "" + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md, buildfolder, metafolder) + def String versionuniquecomponent = '' + def String versionwithpathscomponent = '' for (key in componentReleaseInfoVersionUnique.keySet()) { versionuniquecomponent += "${key}\n" + versionwithpathscomponent += "${key}: path: ${componentReleaseInfoVersionUnique[key].accumulatedpath} meta path: ${componentReleaseInfoVersionUnique[key].accumulatedmetapath}\n" } + echo 'Version unique components:' echo versionuniquecomponent + echo 'Version unique components with paths:' + echo versionwithpathscomponent + if (DO_INTEGRITY_CHECK == "true") { if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct? (If you ignore this issue, you will get an error much later because the system cannot tag 2 different commits of the associated source code with the same label)" @@ -427,7 +432,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - if (!buildinfo.jenkinsbuildinfo.isFullBuild()) { + if (!buildinfo.metainfo.jenkinsbuildinfo.isFullBuild()) { error "The subcomponent: ${buildinfo.componentidentifier()} is not a full build (Some artifacts are missing)" } } @@ -436,8 +441,11 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - if (toplevelid != key) { - releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + echo "Releasing component: ${buildinfo.metainfo.componentidentifier()}" + if (buildinfo.metainfo.componentidentifier() != '___' && buildinfo.metainfo.componentidentifier() != '_') { + if (toplevelid != key) { + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + } } } // Do the top level last diff --git a/release/release.groovy b/release/release.groovy index 351985d..82af511 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -23,18 +23,16 @@ pipeline { stage('Get build artifact meta') { steps { script { - BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" - copyArtifacts( - filter: 'meta/**', - projectName: BUILD_JOB, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR - ], - target: BUILD_ARTIFACTS - ) - def mfs = autobuilderscript.getmetafolderscript() - buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + BUILD_ARTIFACTS = 'build' + + def importoptions = importer.getNewImportOptions() + importoptions.jenkinsjob(BUILD_JOB, BUILD_JOB_SELECTOR) + importoptions.setincrementversionnumber(false) + importoptions.setfilter('meta/**') + importoptions.setappendtoparentmeta(false) + buildmeta = importer.artifactimport(meta, importoptions, BUILD_ARTIFACTS) } } } @@ -44,7 +42,7 @@ pipeline { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" - (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy('', BUILD_ARTIFACTS, buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' @@ -56,7 +54,8 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive("summary/**,${artifactlist}") } } } diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index 5a7c6f5..ae91cb0 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -13,12 +13,6 @@ pipeline { script { configurationhelper = autobuilder.configurationhelper() - PYTHONURL = 'git@hydra:tools/python.git' - PYTHONBRANCH = 'origin/3.6.3_windows' - PYTHONDIR = "${WORKSPACE}/python" - configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR) - env.PYTHON = "${PYTHONDIR}/ver/python.exe" - CCMURL = 'git@hydra:tools/ccm.git' CCMBRANCH = 'origin/master' CCMDIR = "${WORKSPACE}/ccm" @@ -30,49 +24,40 @@ pipeline { stage('Checkout source') { steps { script { - SOURCEREVISION = "${SOURCE_REVISION}" SOURCEDIR = "${WORKSPACE}/source" - configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) - SOURCE_PATH = "${SOURCEDIR}" - env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}" + configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCE_REVISION, SOURCEDIR, '', 0, false) } } } stage("Clean exports") { steps { script { - STATICANALYSIS = 'staticanalysis' + RESULTS = 'results' cleanfolders = [ - STATICANALYSIS, + RESULTS, ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Generating version info') { - steps { - script { - version = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - version.generate(meta, SOURCE_REVISION, SOURCE_PATH) - } - } - } stage('Static analysis') { steps { script { - def CCMOLDER = "${STATICANALYSIS}/ccm" - dir(CCMOLDER) { - echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" + def ccmconfigurationfile = "${WORKSPACE}/ccm.xml" + writeFile(file: ccmconfigurationfile, text: CCM_CONFIGURATION) + echo "Configuration file: ${ccmconfigurationfile} contains:\n${CCM_CONFIGURATION}" + + dir(RESULTS) { + echo "Executing: ${CCM} ${ccmconfigurationfile} > ccm.xml" if (isUnix()) { sh """#!/bin/bash - find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \\; - # Uses mono - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + ${CCM}" "${ccmconfigurationfile}" > ccm.xml """ } else { + echo "Executing: ${WINDOWS_COMMANDLINE_ENVIRONMENT} \"${CCM}\" \"${ccmconfigurationfile}\" > ccm.xml" bat """ - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + ${WINDOWS_COMMANDLINE_ENVIRONMENT} "${CCM}" "${ccmconfigurationfile}" > ccm.xml """ } } @@ -82,25 +67,6 @@ pipeline { } } } - stage('Classifying publishable artifacts') { - steps { - script { - def publishable = [ - ] - publishutils = load("${autobuilder.commonscripts()}/publishing/utils.groovy") - publishutils.generatelist(meta, publishable) - } - } - } - stage('Archiving artifacts') { - steps { - script { - ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] - def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts artifactlist - } - } - } } post { always { diff --git a/staticanalysis/cpd.groovy b/staticanalysis/cpd.groovy index 5a77cab..c04649c 100644 --- a/staticanalysis/cpd.groovy +++ b/staticanalysis/cpd.groovy @@ -129,7 +129,8 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts artifactlist + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index 9455b0b..d959770 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -91,7 +91,8 @@ pipeline { script { ArrayList itemsForRelease = ["${STATICANALYSIS}/**"] def artifactlist = meta.generateartifacts(itemsForRelease) - archiveArtifacts artifactlist + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/test/testethernetproduct.groovy b/test/testethernetproduct.groovy index 92d6414..5e2ce53 100644 --- a/test/testethernetproduct.groovy +++ b/test/testethernetproduct.groovy @@ -192,7 +192,8 @@ pipeline { steps{ script { def artifactlist = meta.generateartifacts([ "${LOGGING_ROOT}/**"]) - archiveArtifacts artifactlist + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + artifactarchiver.archive(meta, artifactlist) } } } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 50f4025..a22054b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -194,14 +194,14 @@ def docheckout(String gitURL, String revision, int cloneDepth, String reporootpa branches: [ [name: revision] ], - doGenerateSubmoduleConfigurations: false, extensions: [ - [$class: 'GitLFSPull',timeout: 25], - [$class: 'CleanBeforeCheckout',timeout: 20], + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + [$class: 'PruneStaleBranch'], ], submoduleCfg: [], ]) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 6e0db11..5e974e2 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -98,6 +98,9 @@ def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { def integerstrings = sourcebranch.findAll(/\d+/) def integers = [] for (integerstring in integerstrings) { + if (integerstring.length() > 6) { + integerstring = integerstring.substring(0, 5) + } int integer = integerstring.toInteger() integer = integer % (maxversionnumber() + 1) integers.add(integer) @@ -154,6 +157,13 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +String pregenerated(meta, String sourcerevision, String sourcepath, String pregeneratedbuildstamp, int basebuildnumber = 1000) { + generate(meta, sourcerevision, sourcepath, basebuildnumber) + meta.buildstampinfo.override(pregeneratedbuildstamp) + + return pregeneratedbuildstamp +} + String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { def int bbn = basebuildnumber.trim().toInteger() return generate(meta, sourcerevision, sourcepath, bbn) From be960001c4711a0ac9c703364b0673f9a38147b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 30 Jan 2022 19:20:56 +0000 Subject: [PATCH 11/15] Squashed 'autobuild/Common_scripts/' changes from 6f0a448..0326879 0326879 Eliminate a copy for performance cbc3289 Eliminate a copy for performance 3271d90 Eliminate a copy for performance 4044992 Eliminate a copy for performance 2e74e8a Eliminate a copy for performance 10d0019 Ensure script objects are around when source files have been cleaned 1e29bd5 Force item copier workspace to be cleaned each time 5575f2b Don't attempt to import artifacts associated with components that do not have complete meta 9b89c0f CORE-348 - Reverse a debugging change 3db76ce Merge remote-tracking branch 'origin/master' into CORE-348 2a17eb0 Merge commit 'ad8723e7c8df7b9ffceec3540fd88e9f847e7a7f' into production b9f5240 / missing when doing hierarchical release repo checkout e67bec1 When doing nested release repo imports - ignore components that have incomplete release info 2ab8282 Merge commit 'ea3a620531419dd99f369c664f7fb04291dfe2c0' into master 8e15e7b CORE-594 - Import artifacts using full name 951032d CORE-594 - Debug installer creation 6a415af CORE-594 - Debug installer creation cdd00fd CORE-594 - Debug installer creation eda4801 CORE-594 - Debug installer creation fb721bb CORE-594 - Debug installer creation f1987cb CORE-594 - Debug installer creation d282099 CORE-594 - Debug installer creation 05a32d0 CORE-594 - Debug installer creation d70f633 CORE-594 - Debug installer creation 945f837 CORE-594 - Debug installer creation 36ed63e CORE-594 - Debug installer creation a1c2d3b CORE-594 - Debug installer creation 5b3f7a7 CORE-594 - Debug installer creation cfcdf5c CORE-594 - Debug installer creation f821da7 CORE-594 - Debug installer creation 2c26b22 Correct hierarchical import of artifacts from release repositories 0c87c2b Correct hierarchical import of artifacts from release repositories aaa7b8a Correct hierarchical import of artifacts from release repositories 1a0dcf9 Correct hierarchical import of artifacts from release repositories 00afceb Correct hierarchical import of artifacts from release repositories b365f5b Correct hierarchical import of artifacts from release repositories eb72549 Correct hierarchical import of artifacts from release repositories 5c3c949 Correct hierarchical import of artifacts from release repositories 8be38b8 Correct hierarchical import of artifacts from release repositories 51224e0 Correct hierarchical import of artifacts from release repositories 9d04525 Correct hierarchical import of artifacts from release repositories 16d7c9c Merge remote-tracking branch 'origin/master' into CORE-348 56951ba CORE-348: Trying to resolve a git error in the autobuild. git-subtree-dir: autobuild/Common_scripts git-subtree-split: 0326879391035840f42f5e1eb5ffc92e834c8995 --- artifacts/metaexport.groovy | 2 +- copy/all.groovy | 36 +++++--------- import/importer.groovy | 92 ++++++++++++++++++++++++++---------- install/installhelper.groovy | 25 ++++++++-- meta/folder.groovy | 25 +++++++--- 5 files changed, 121 insertions(+), 59 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 8e5e56e..a40e9a8 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,7 +4,7 @@ def combineSubjobArtifacts(meta, importer, subJobs) { for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { - def subJobName = subJobObject.getProjectName() + def subJobName = subJobObject.getFullProjectName() def subJobNumber = subJobObject.getNumber() def destFolder = subJob[1] def boolean incrementversionnumber = subJob[2] diff --git a/copy/all.groovy b/copy/all.groovy index aab479c..24aacfb 100644 --- a/copy/all.groovy +++ b/copy/all.groovy @@ -4,21 +4,13 @@ pipeline { stage('Initial setup') { steps { script { + // Force workspace to be cleaned + CLEAN_WORKSPACE = "true" meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - } - } - } - stage('Clean exports') { - steps { - script { - IMPORTS = 'imports' - cleanfolders = [ - IMPORTS, - ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") - cleaner.clean(cleanfolders) + artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") } } } @@ -34,29 +26,25 @@ pipeline { SOURCE_ARTIFACT_RELEASE_REVISION) echo "Nested: ${NESTED_COPY}" copyoptions.setnested(NESTED_COPY == 'true') - importer.artifactimport(meta, copyoptions, IMPORTS) + importer.artifactimport(meta, copyoptions, "${WORKSPACE}/.") } } } stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(['**/*']) - def artifactarchiver = load("${autobuilder.commonscripts()}/artifacts/archiver.groovy") + // Save the imported meta - while the unwanted meta is written to file then deleted + def stashname = 'Imported meta stash' + stash(allowEmpty: true, + includes: 'meta/**', + name: stashname, + useDefaultExcludes: false) + def artifactlist = meta.generateartifacts(['**/*']) cleaner.clean(['meta', 'autobuild']) - dir(IMPORTS) { - fileOperations([ - fileCopyOperation(excludes: '', - flattenFiles: false, - includes: '**/*', - targetLocation: WORKSPACE) - ]) - } + unstash(name: stashname) - cleaner.clean([IMPORTS]) - artifactarchiver.archive(meta, artifactlist) } } diff --git a/import/importer.groovy b/import/importer.groovy index 8bb565a..fa32ebf 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -20,6 +20,8 @@ class ImportOptions { String m_repo String m_revision + + boolean m_showdetails ImportOptions(script, String sourceselect = 'Jenkins job artifact', @@ -31,7 +33,8 @@ class ImportOptions { boolean flatten = false, boolean includemetainfilter = true, boolean clean = true, - appendtoparentmeta = true) { + appendtoparentmeta = true, + showdetails = false) { m_script = script m_sourceselect = sourceselect.trim() m_incrementversionnumber = incrementversionnumber @@ -43,6 +46,25 @@ class ImportOptions { m_includemetainfilter = includemetainfilter m_clean = clean m_appendtoparentmeta = appendtoparentmeta + m_showdetails = showdetails + } + + String show() { + return """m_incrementversionnumber: ${m_incrementversionnumber} + m_filter: ${m_filter} + m_excludes: ${m_excludes} + m_fingerprint: ${m_fingerprint} + m_nested: ${m_nested} + m_flatten: ${m_flatten} + m_includemetainfilter: ${m_includemetainfilter} + m_clean: ${m_clean} + m_appendtoparentmeta: ${m_appendtoparentmeta} + m_sourceselect: ${m_sourceselect} + m_jobname: ${m_jobname} + m_buildnumberorselector: ${m_buildnumberorselector} + m_repo: ${m_repo} + m_revision: ${m_revision} + """ } void setsourceselect(String sourceselect) { @@ -56,7 +78,7 @@ class ImportOptions { m_sourceselect = sourceselect break default: - error("Unknown type of artifact source select: ${m_sourceselect}") + m_script.error("Unknown type of artifact source select: ${sourceselect}") } } @@ -96,7 +118,7 @@ class ImportOptions { m_appendtoparentmeta = appendtoparentmeta } - void jenkinsjob(String jobname, buildnumberorselector) { + void jenkinsjob(String jobname, String buildnumberorselector) { m_jobname = jobname.trim() m_buildnumberorselector = [ $class: 'ParameterizedBuildSelector', @@ -109,7 +131,7 @@ class ImportOptions { m_jobname = jobname.trim() m_buildnumberorselector = [ $class: 'ParameterizedBuildSelector', - parameterName: "${buildnumberorselector}" + parameterName: "${buildnumberorselector}" ] setsourceselect('Jenkins job artifact') } @@ -125,6 +147,10 @@ class ImportOptions { releaserepo(repo, revision) setsourceselect(sourceselect) } + + void showdetails(boolean showdetails) { + m_showdetails = showdetails + } } ImportOptions getNewImportOptions() { @@ -151,7 +177,7 @@ def conditionalClean(ImportOptions importoptions, String destpath) { } } -def artifactimportfrombuildjob(meta, importoptions, String destpath) { +def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_sourceselect != 'Jenkins job artifact') { error("Import options settings have not been set for: Jenkins job artifact") return @@ -183,7 +209,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { +def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_sourceselect != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -199,24 +225,31 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) - - relatedimportinfo = importedmeta.getassociated() - echo "Cloning associated repos" - for(relatedmetainfo in relatedimportinfo) { - def relatedmeta = relatedmetainfo.meta - def relatedoffsetpath = relatedmetainfo.offsetpath + + echo "Getting associated metadata from path: ${destpath}" + def collectedmetalist = importedmeta.getassociated(destpath) + for(def collectedmeta in collectedmetalist) { + echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}" + } + + echo "Cloning associated repos inside path: ${destpath}" + for(def collectedmeta in collectedmetalist) { + def relatedmeta = collectedmeta.metainfo + def relatedoffsetpath = collectedmeta.accumulatedpath def cloneurl = relatedmeta.releaseinfo.releaserepoURL def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" - echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - - gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) - - gh.stripgithistory(clonepath) - - def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" - echo "Marking related meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(relatedmeta, metafilepath) + if (relatedmeta.componentidentifier() != '_' && + cloneurl != '') { + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } else { + echo "Not importing component that would be placed at path: ${clonepath}" + } } gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" @@ -229,7 +262,7 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { return importedmeta } -def artifactimportfromrepo(meta, importoptions, String destpath) { +def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) { if (importoptions.m_sourceselect != 'Release repository') { error("Import options settings have not been set for: Release repository") return @@ -257,7 +290,10 @@ def artifactimportfromrepo(meta, importoptions, String destpath) { return importedmeta } -def artifactimport(meta, importoptions, String destpath) { +def artifactimport(meta, ImportOptions importoptions, String destpath) { + if (importoptions.m_showdetails) { + echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}" + } switch (importoptions.m_sourceselect) { case 'Jenkins job artifact': case 'Jenkins job artefact': @@ -342,7 +378,13 @@ def artifactimport(meta, } // Import the artifacts of a named child component found below parent meta's hierarchy -def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, String childsubcomponent, importoptions, String destinationfolder) { +def artifactimport(meta, + parentmeta, + String parentmetapath, + String childproduct, + String childsubcomponent, + ImportOptions importoptions, + String destinationfolder) { // Import from parent meta def childmeta = meta.getNewMetaObject() def childmeta2 @@ -351,7 +393,7 @@ def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) if (findresults.matchfound) { childmeta = findresults.matchedmetaitem.metainfo - if (childmeta.releaseinfo.released) { + if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) { importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) } else { importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 96a8b2e..8dbf23e 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -23,17 +23,32 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin } // Given details about artifacts to import and details of installers to build, import artifacts that are required -def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) { artifactFetched = [:] + if (showdetails) { + echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}") + } def importoptions = importer.getNewImportOptions() + importoptions.showdetails(showdetails) for (installer in toBuild) { - def artifactKeys = installerDetails.get(installer)[1] + def installerinfo = installerDetails.get(installer) + if (showdetails) { + echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}") + } + def artifactKeys = installerinfo[1] if (artifactKeys instanceof java.lang.String) { artifactKeys = [ artifactKeys ] } + if (showdetails) { + echo ("artifactKeys:\n${artifactKeys}") + } for (artifactKey in artifactKeys) { - (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + artifactDetail = artifactDetails.get(artifactKey) + if (showdetails) { + echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}") + } + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail if (artifactFetched.get(artifactKey) == null) { importoptions.jenkinsjob(sourceProject, sourceselector) importoptions.setincrementversionnumber(incrementversionnumber) @@ -45,6 +60,9 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { } } } + if (showdetails) { + echo ("artifactFetched:\n${artifactFetched}") + } return artifactFetched } @@ -93,6 +111,7 @@ def checkISMfilebackend(String xml) { def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) + //echo "ISM file xml: ${xml}" (exitcode, message) = checkISMfilebackend(xml) echo message if (exitcode > 0) { diff --git a/meta/folder.groovy b/meta/folder.groovy index b9e898c..6715145 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -608,6 +608,7 @@ class MetaInfo extends GISJsonSerialised { abstract void deserialiseJson(rawmeta) abstract LinkedHashMap serialisationdata() abstract String getrelatedid() + abstract void markreleased() } class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { @@ -642,6 +643,10 @@ class MetaInfo extends GISJsonSerialised { String getrelatedid() { return uniqueid } + + void markreleased() { + meta.releaseinfo.markreleased() + } } class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { @@ -674,6 +679,10 @@ class MetaInfo extends GISJsonSerialised { String getrelatedid() { return uniqueid } + + void markreleased() { + // Do nothing + } } private ProductInfo productinfo @@ -701,15 +710,11 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } - List getassociated() { - return associatedmeta - } - void markreleased() { script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") releaseinfo.markreleased() for (associated in associatedmeta) { - associated.meta.releaseinfo.markreleased() + associated.markreleased() } } @@ -1079,6 +1084,7 @@ class MetaInfo extends GISJsonSerialised { return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] } + // Walks all layers of associated data only def allrelatedmeta(String metapath = '') { def ArrayList metacollection = new ArrayList() def Map metamap = new LinkedHashMap() @@ -1088,10 +1094,17 @@ class MetaInfo extends GISJsonSerialised { ArrayList releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { def ArrayList metacollection = new ArrayList() def Map metamap = new LinkedHashMap() - def results = walkrelated(new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth)) + def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth) + script.echo("Release walker iteration info at start:\n${ii.show()}") + def results = walkrelated(ii) return results.metacollection } + // Walks 1 layer of associated data only + def getassociated(String rootpath) { + return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself) + } + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, CollectedMeta metaitem) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] From e106bb64ef32df8431789a11d22d3d422cb9d651 Mon Sep 17 00:00:00 2001 From: Dave Steele Date: Fri, 11 Mar 2022 14:43:23 +0000 Subject: [PATCH 12/15] ELGIP-186. tb_ctrl.vhd outputs 3 clocks which are delta times appart. Fix so that in sync as per a PLL so SIM matches real world --- verification/tb_template/test/tb_ctrl.vhd | 45 ++++++++++++++--------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/verification/tb_template/test/tb_ctrl.vhd b/verification/tb_template/test/tb_ctrl.vhd index a1975e2..eeb4d09 100644 --- a/verification/tb_template/test/tb_ctrl.vhd +++ b/verification/tb_template/test/tb_ctrl.vhd @@ -36,20 +36,20 @@ port ( clk : out std_logic := '1'; clk_div2 : out std_logic := '1'; rst_n : out std_logic := '0'; - + count_vec : out std_logic_vector(31 downto 0); timeout : in std_logic_vector(31 downto 0); - + -- Simulation status and control inputs error : in std_logic; do_stop : in std_logic; do_finish : in std_logic; partial_test : in std_logic; - + -- External end-of-simulation flags failed : out boolean; passed : out boolean - + ); end entity; @@ -57,7 +57,11 @@ architecture sim of tb_ctrl is package tb_pkg is new work.tb_pkg; use tb_pkg.all; - + + signal clk_x2_e2 : std_logic := '1'; + signal clk_x2_e1 : std_logic := '1'; + signal clk_e1 : std_logic := '1'; + signal is_gui_run : boolean := false; signal count : integer := 0; signal end_sim : std_logic; @@ -69,26 +73,31 @@ begin ----------------------------------------- -- Clock generation with concurrent procedure call - clk_gen(clk_x2, Real(CLK_FREQ_MHZ) * 2.0E6); -- x2 clock - + clk_gen(clk_x2_e2, Real(CLK_FREQ_MHZ) * 2.0E6); -- x2 clock + -- Export counter count_vec <= std_logic_vector(to_unsigned(count, 32)); -- Derive system clock from master clock - process(clk_x2) + process(clk_x2_e2) begin - if clk_x2'event and clk_x2 = '1' then - clk <= not clk; + if clk_x2_e2'event and clk_x2_e2 = '1' then + clk_e1 <= not clk_e1; end if; end process; - process(clk) + process(clk_e1) begin - if clk'event and clk = '1' then + if clk_e1'event and clk_e1 = '1' then clk_div2 <= not clk_div2; end if; end process; + -- delta time corrections. All clocks should be in phase as per a PLL + clk_x2_e1 <= clk_x2_e2; + clk_x2 <= clk_x2_e1; + clk <= clk_e1; + -- Generate a reset process (clk) begin @@ -97,14 +106,14 @@ begin rst_n <= '1' when count > 10 else '0'; end if; end process ; - + -- End-of-simulation signals end_sim <= '1' when count = to_integer(unsigned(timeout)) or do_stop = '1' or do_finish = '1' else '0'; failed <= true when end_sim = '1' and (error = '1' or count = to_integer(unsigned(timeout))) else false; passed <= true when end_sim = '1' and error = '0' else false; - + ----------------------------------------- - -- Simulation control from VProc over + -- Simulation control from VProc over -- CSR bus ----------------------------------------- @@ -128,7 +137,7 @@ begin if error = '1' then report "*** FAIL: errors found" severity failure; - + else if count >= to_integer(unsigned(timeout)) then report "*** FAIL: timeout" severity failure; @@ -138,7 +147,7 @@ begin report "SUCCESS" severity note; end if; end if; - + if is_gui_run or (do_stop = '1' and do_finish = '0') then stop(0); else @@ -149,4 +158,4 @@ begin end if; end process; -end architecture; \ No newline at end of file +end architecture; From 502eaa07184b504cb7a4da4d88b9e710591f2370 Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 5 Apr 2022 15:41:35 +0100 Subject: [PATCH 13/15] HMBKYM16ST-38 Add "bitdef" as a new legal type for fields. Used to handover bit definitions to ESW --- logic_memory_mapper/memMapperBase.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/logic_memory_mapper/memMapperBase.py b/logic_memory_mapper/memMapperBase.py index 16d3cb1..52425ae 100644 --- a/logic_memory_mapper/memMapperBase.py +++ b/logic_memory_mapper/memMapperBase.py @@ -62,8 +62,9 @@ class regtype : CONSTANT = 'c' RESERVED = 'rsv' READ_WRITE_PULSE = 'rwp' + BIT_DEFINITION = 'bitdef' - typeList = [WRITE, WRITE_PULSE, WRITE_CLR0, READ, READ_PULSE, CONSTANT, RESERVED, READ_WRITE_PULSE] + typeList = [WRITE, WRITE_PULSE, WRITE_CLR0, READ, READ_PULSE, CONSTANT, RESERVED, READ_WRITE_PULSE, BIT_DEFINITION] # --------------------------------------------------------- # Abstract base class to locate JSON descriptions of @@ -81,7 +82,7 @@ class MemMapperBase : # used in validation of model writeTypes = [regtype.WRITE, regtype.WRITE_CLR0, regtype.WRITE_PULSE, regtype.READ_WRITE_PULSE] readTypes = [regtype.READ, regtype.READ_PULSE, regtype.CONSTANT] - neutralTypes = [regtype.RESERVED] + neutralTypes = [regtype.RESERVED, regtype.BIT_DEFINITION] noReadTypes = [regtype.WRITE_CLR0, regtype.RESERVED] wrPulseTypes = [regtype.WRITE_PULSE, regtype.READ_WRITE_PULSE] From 55387dc256c0a50454964cbf696f14bc5218d9da Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 5 Apr 2022 15:43:40 +0100 Subject: [PATCH 14/15] HMBKYM16ST-38 Change any bit fields that are single bit from "subtype with range " to "constant". Subtype with range is not used anywhere --- logic_memory_mapper/gen_pkg.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/logic_memory_mapper/gen_pkg.py b/logic_memory_mapper/gen_pkg.py index 2570f78..79efff3 100644 --- a/logic_memory_mapper/gen_pkg.py +++ b/logic_memory_mapper/gen_pkg.py @@ -208,9 +208,17 @@ class MemMapperPkg(MemMapperBase) : # Pad the string to match the desired o/p width fieldFullName = MemMapperPkg.__padString(fieldFullName, MemMapperPkg.outputNameWidth) - # Output the bit-slice subtype declaration - wfp.write(' subtype ' + fieldFullName + - ' is natural range ' + topStr + ' downto ' + bottomStr + ';\n') + if fieldLen == 1: + + # Output single bit as a constant + wfp.write(' constant ' + fieldFullName + + ' : integer := ' + topStr + ';\n') + + else: + + # Output the bit-slice subtype declaration + wfp.write(' subtype ' + fieldFullName + + ' is natural range ' + topStr + ' downto ' + bottomStr + ';\n') # Put a blank line between busses wfp.write('\n') From 586f590120312be2ecede8133194fe9a0789092d Mon Sep 17 00:00:00 2001 From: David Steele Date: Tue, 5 Apr 2022 15:44:47 +0100 Subject: [PATCH 15/15] HMBKYM16ST-38 Identify registers used for bit definitions to ESW and mark them as unusable --- logic_memory_mapper/gen_htm.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/logic_memory_mapper/gen_htm.py b/logic_memory_mapper/gen_htm.py index 848acb2..e3fe0b8 100644 --- a/logic_memory_mapper/gen_htm.py +++ b/logic_memory_mapper/gen_htm.py @@ -82,7 +82,7 @@ class MemMapperHtm(MemMapperBase): # --------------------------------------------------------- def setWidth(self, width): self.__tableWidth = width - + # --------------------------------------------------------- # Configure the notes width # --------------------------------------------------------- @@ -181,6 +181,9 @@ class MemMapperHtm(MemMapperBase): notesStr += 'rwpSimilar to wp, but also generates a (separate) pulse on a read access to the register\n' notesStr += '\n' notesStr += '\n' + notesStr += 'bitdefDefines the bits used in the read, set, clear registers\n' + notesStr += '\n' + notesStr += '\n' notesStr += '  \n' notesStr += '\n' notesStr += '\n' @@ -463,7 +466,7 @@ class MemMapperHtm(MemMapperBase): regType = regDict[regName][attr.TYPE] else: regType = ' ' - + if attr.SUBTYPE in regDict[regName] : regSubType = regDict[regName][attr.SUBTYPE] else : @@ -485,6 +488,17 @@ class MemMapperHtm(MemMapperBase): # Generate a hex string for the value (with a leading '0x') addrStr = '0x' + format(addrVal, 'x') + rowSpan = 1 + hasFields = attr.FIELDS in regDict[regName] + + if hasFields : + fields = regDict[regName][attr.FIELDS] + rowSpan = len(fields) + 1 + + # Bitdefs are not real registers so mark as N/A + if fields[list(fields)[0]][attr.TYPE] == regtype.BIT_DEFINITION: + addrStr = 'N/A' + # Gather together each column's string into a list for this register entryList = [regNameStr, addrStr, @@ -495,13 +509,6 @@ class MemMapperHtm(MemMapperBase): reset, regDict[regName][attr.DESCRIPTION]] - rowSpan = 1 - hasFields = attr.FIELDS in regDict[regName] - - if hasFields : - fields = regDict[regName][attr.FIELDS] - rowSpan = len(fields) + 1 - # Draw the row for the register self.__drawContents(entryList, wfp, False, rowSpan, not hasFields, lastReg) @@ -520,7 +527,7 @@ class MemMapperHtm(MemMapperBase): resetVal = '0' else : resetVal = fields[fieldName][attr.RESET] - + if attr.SUBTYPE in fields[fieldName] : regSubType = fields[fieldName][attr.SUBTYPE] else :