From 95caf56875baa6ed8b58fd36c7e9cf9855a3ea81 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 13:28:28 +0000 Subject: [PATCH 01/31] Make installer build use meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 643e203..15f77d0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -605,8 +605,8 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), - ArrayList releasedfromsubmeta = new ArrayList(), + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { def String artifactlist = METAARTIFACTS From a5cd7e43acf791622620f270e12decfe2953ff3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 14:47:13 +0000 Subject: [PATCH 02/31] Import specific job binaries; Add imported meta to this jobs meta --- import/importer.groovy | 6 ++++-- install/installhelper.groovy | 19 +++++-------------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..bf040ce 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,7 +1,8 @@ -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, + excludes: excludes, fingerprintArtifacts: fingerprint, projectName: jenkinsjob.trim(), selector: specific(buildnumber.toString()), @@ -10,9 +11,10 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St return meta.importrelated(destpath) } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, + excludes: excludes, fingerprintArtifacts: fingerprint, projectName: jenkinsjob.trim(), selector: [ diff --git a/install/installhelper.groovy b/install/installhelper.groovy index f523799..fdd6016 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,30 +1,21 @@ // Given details about artifacts to import and details of installers to build, import artifacts that are required -def artifactsImporter(artifactDetails, installerDetails) { +def artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] - //echo "To build: ${installer} use artifactKey: ${artifactKey}" - (sourceProject, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { - //echo "Artifact info: ${artifactInfo}" - copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" - def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" - def folderToCopy = "${destFolder}\\version" - echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" - fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) - //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) - def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - artifactVersion = gh.getSHAFromVersion("${destFolder}/version") - artifactFetched.put(artifactKey, artifactVersion) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destpath, toCopy, notToCopy, true) } } return artifactFetched } // Export any required binaries to the /bin folder -def exportBinaries(artifactDetails, installerDetails) { +def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { From 5aae28adceb6c3d13281be015d9f9519b3e34262 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 15:06:26 +0000 Subject: [PATCH 03/31] Import specific job binaries; Add imported meta to this jobs meta --- install/installhelper.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index fdd6016..206a6c1 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -6,9 +6,8 @@ def artifactsImporter(meta, artifactDetails, installerDetails) { for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) - env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destpath, toCopy, notToCopy, true) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) } } return artifactFetched From 2d679105c8c4f23cf7e46940d8547c36d9e99351 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 3 Dec 2018 15:36:06 +0000 Subject: [PATCH 04/31] Import specific job binaries; Add imported meta to this jobs meta --- install/installhelper.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 206a6c1..0155c7f 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -1,5 +1,5 @@ // Given details about artifacts to import and details of installers to build, import artifacts that are required -def artifactsImporter(meta, artifactDetails, installerDetails) { +def Map artifactsImporter(meta, artifactDetails, installerDetails) { artifactFetched = [:] def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") @@ -8,6 +8,7 @@ def artifactsImporter(meta, artifactDetails, installerDetails) { (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy, true) + artifactFetched.put(artifactKey, importmeta) } } return artifactFetched From db6163fff766bbaecc039aa3ccafa9df7b2b0aee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Dec 2018 18:30:15 +0000 Subject: [PATCH 05/31] Add publishing script --- publish/do.groovy | 313 ++++++++++++++++++++++++++++++++++++++ versioncontrol/git.groovy | 18 ++- 2 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 publish/do.groovy diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..2c5a018 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,313 @@ +markunstable = false +unstablereason = "" + +FTP_CLIENT_LOCATION='ftp' +FTP_URL='ie1.hostedftp.com' +FTP_USER='ftp@globalinkjetsystems.com' +FTP_CUSTOMER_ROOT='/Customers/' +FTP_PASSWORD='ramshorn42' +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } +} + +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.meta + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" + def publishableitems = releasemeta.publishableinfo + def ArrayList publishablePaths = new ArrayList() + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean changed = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + return changed +} + +def ArrayList maketaglist(String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + taglist.add(othertagitem.trim()) + } + } + return taglist +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def publish(releasemeta, String uniquefolderforrepo) { + summaryappend(" Git operations: ") + def String buildstamp = releasemeta.buildstampinfo.buildstamp + + dir(uniquefolderforrepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + releasegitadd() + + def boolean changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamp) + } + else { + releasegitreset() + summaryappend("No-commit,") + } + + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } +} + +def recallcomponent(releaseinfo) { + def releasemeta = releaseinfo.meta + def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + + summaryappend("\n") +} + +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = releasemeta.releasewalkerunique(md) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + recallcomponent(releaseinfo) + } + // publish(releasemeta, uniquefolderforrepo) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index cb9354b..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,3 +1,7 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { @@ -6,7 +10,16 @@ String getGitInfo(String workingSetRootPath) { } } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +@NonCPS +def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return new SparseCheckoutPaths(sparsecheckoutpaths) +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRoot != "") { reporootpath = "${reporootenv.trim()}/${repoRoot}" @@ -28,7 +41,8 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + sparsecheckoutconfig(sparseitemlist) ], submoduleCfg: [], ]) From 9ea146b13318e6f5beedd7dd917d6cda79ac60b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 11:22:13 +0000 Subject: [PATCH 06/31] Only checkout repositories that have publishable items --- publish/do.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 2c5a018..c7084d0 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,11 +104,13 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" def publishableitems = releasemeta.publishableinfo def ArrayList publishablePaths = new ArrayList() - for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } def releasegitadd() { From a7f6f97d7720cebc81be274449a9634d3569cfbc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:15:10 +0000 Subject: [PATCH 07/31] Build up folder structure for FTP --- publish/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c7084d0..4d02b4f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -281,9 +281,9 @@ def publish(releasemeta, String uniquefolderforrepo) { } } -def recallcomponent(releaseinfo) { +def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -305,7 +305,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo) + recallcomponent(releaseinfo, customer, softwarerelease) } // publish(releasemeta, uniquefolderforrepo) From 67ca5024c206ccfd03e322717df27ae1458332f9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:59:24 +0000 Subject: [PATCH 08/31] Change root path for transfer; Add an example FTP operation --- publish/do.groovy | 53 ++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 4d02b4f..b953f4c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,11 +1,6 @@ markunstable = false unstablereason = "" -FTP_CLIENT_LOCATION='ftp' -FTP_URL='ie1.hostedftp.com' -FTP_USER='ftp@globalinkjetsystems.com' -FTP_CUSTOMER_ROOT='/Customers/' -FTP_PASSWORD='ramshorn42' summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" @@ -250,40 +245,32 @@ def sourcegitpush(String dryrunpush ) { } } -def publish(releasemeta, String uniquefolderforrepo) { - summaryappend(" Git operations: ") - def String buildstamp = releasemeta.buildstampinfo.buildstamp +def String FTP_CLIENT_LOCATION='ftp' +def String FTP_URL='ie1.hostedftp.com' +def String FTP_USER='ftp@globalinkjetsystems.com' +def String FTP_CUSTOMER_ROOT='/Customers/' +def String FTP_PASSWORD='ramshorn42' - dir(uniquefolderforrepo) { - def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Pre commit sha code: ${precommitshacode}" - - releasegitadd() +def publish() { + summaryappend(" FTP operations: ") - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamp) - } - else { - releasegitreset() - summaryappend("No-commit,") - } - - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" - - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - def tagadded = releasegittags(postcommitshacode, taglist) - - if (changed || tagadded) { - releasegitpush(dryrunpush) - } + dir(releaserepofolder) { + sh """#!/bin/bash + ftp -n -v ${FTP_URL} << EOT + ascii + user ${FTP_USER} ${FTP_PASSWORD} + prompt + cd ${FTP_CUSTOMER_ROOT} + ls -la + bye + EOT + """ } } def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -307,7 +294,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, customer, softwarerelease) } - // publish(releasemeta, uniquefolderforrepo) + publish() return [summarystring, markunstable, unstablereason] } From b178b86f6ee9199bdc418f5cf79d28b6505b7a00 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 14:26:20 +0000 Subject: [PATCH 09/31] Build up folder structure for FTP --- publish/do.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b953f4c..46b3fdd 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -245,15 +245,15 @@ def sourcegitpush(String dryrunpush ) { } } -def String FTP_CLIENT_LOCATION='ftp' -def String FTP_URL='ie1.hostedftp.com' -def String FTP_USER='ftp@globalinkjetsystems.com' -def String FTP_CUSTOMER_ROOT='/Customers/' -def String FTP_PASSWORD='ramshorn42' def publish() { summaryappend(" FTP operations: ") + def String FTP_CLIENT_LOCATION='ftp' + def String FTP_URL='ie1.hostedftp.com' + def String FTP_USER='ftp@globalinkjetsystems.com' + def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { sh """#!/bin/bash ftp -n -v ${FTP_URL} << EOT From 4230e4dcd33918892e2f4ac90fafb688dad4ba87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:14:29 +0000 Subject: [PATCH 10/31] Use ncftpput to transfer --- publish/do.groovy | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 46b3fdd..13a280a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -106,6 +106,9 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } + else { + summaryappend("None ") + } } def releasegitadd() { @@ -245,32 +248,26 @@ def sourcegitpush(String dryrunpush ) { } } - -def publish() { +def publish(String customer) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' def String FTP_URL='ie1.hostedftp.com' def String FTP_USER='ftp@globalinkjetsystems.com' - def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { + summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") sh """#!/bin/bash - ftp -n -v ${FTP_URL} << EOT - ascii - user ${FTP_USER} ${FTP_PASSWORD} - prompt - cd ${FTP_CUSTOMER_ROOT} - ls -la - bye - EOT + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ + summaryappend(", Transferred ") } } -def recallcomponent(releaseinfo, String customer, String softwarerelease) { +def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -292,9 +289,9 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo, customer, softwarerelease) + recallcomponent(releaseinfo, softwarerelease) } - publish() + publish(customer) return [summarystring, markunstable, unstablereason] } From e06d3705497981903880f07cb750004a800e5669 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:59:04 +0000 Subject: [PATCH 11/31] Use ncftpput to transfer --- publish/do.groovy | 200 +++------------------------------------------- 1 file changed, 10 insertions(+), 190 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 13a280a..8cdf9a3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -5,12 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -42,54 +37,7 @@ def summaryappend(String summary) { summarystring += summary } -def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = buildmeta.getNewMetaObject(this) - // See if there is meta data file from the current release - def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" - if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(checkoutfolder) - } - return previousreleasemeta -} - -def checkoutreleaserepo(buildmeta, String checkoutfolder) { - def String releaserepo = buildmeta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - def previousreleasemeta = buildmeta.getNewMetaObject(this) - dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") - } - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) - dir(checkoutfolder) { - // Clear all existing files - sh("git rm -r *") - } - return previousreleasemeta -} - -def checkoutsourcerepo(buildmeta, String checkoutfolder) { - def String sourcerepo = buildmeta.gitinfo.remoteurl - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } -} - +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -111,143 +59,6 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } } -def releasegitadd() { - // Add anything new - sh("git add .") - summaryappend("Add,") -} - -def releasegitreset() { - // Add anything new - sh("git reset --hard") - sh("git clean -fdx") - summaryappend("Reset,") -} - -def releasegitcommit(String buildstamp) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") -} - -// Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { - String buildsha = buildmeta.gitinfo.gitrevision - String previoussha = previousreleasemeta.gitinfo.gitrevision - - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return changed -} - -def ArrayList maketaglist(String buildstamp, String otherreleasetags) { - def ArrayList taglist = new ArrayList() - taglist.add(buildstamp) - - def ArrayList othertaglist = otherreleasetags.split(',') - for (othertagitem in othertaglist) { - if (othertagitem?.trim()) { - taglist.add(othertagitem.trim()) - } - } - return taglist -} - -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\"") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def boolean releasegittags(String postcommitshacode, ArrayList taglist) { - boolean tagadded = false - - summaryappend("Rel Tag ") - for (tag in taglist) { - if (releasegittag(postcommitshacode, tag)) { - tagadded = true - } - } - return tagadded -} - -def boolean sourcegittag(String shacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != shacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\" ${shacode}") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def sourcegittags(String buildsha, List taglist) { - boolean tagadded = false - - summaryappend(",Src Tag ") - for (tag in taglist) { - if (sourcegittag(buildsha, tag)) { - tagadded = true - } - } - return tagadded -} - -def releasegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend(",Dry-Run-Push") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend(",Push") - } -} - -def sourcegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags") - summaryappend(",Src-Dry-Run-Push") - setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags") - summaryappend(",Src Push") - } -} - def publish(String customer) { summaryappend(" FTP operations: ") @@ -278,6 +89,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { summaryappend("\n") } +def stripgithistory() { + dir(releaserepofolder) { + sh """#!/bin/bash + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + """ + } +} + def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() @@ -291,6 +110,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, softwarerelease) } + stripgithistory() publish(customer) return [summarystring, markunstable, unstablereason] From 8ba5a2c59a7407c8d993bd343793818953edbc7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 16:01:13 +0000 Subject: [PATCH 12/31] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 8cdf9a3..34d0e42 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -68,7 +68,7 @@ def publish(String customer) { def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { - summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") + summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ From 08e458d9d633bfca42d0dd39bb5e549066a6b277 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:12:45 +0000 Subject: [PATCH 13/31] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 34d0e42..a30bfc9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -92,7 +92,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf """ } } From 64a6b177de8bce34c2a37982ec5305bd62be2d96 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:53:08 +0000 Subject: [PATCH 14/31] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- artifacts/metaexport.groovy | 8 ++------ import/importer.groovy | 10 ++++++++-- versionnumbering/generate.groovy | 5 +++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b4d4a6d..0ca5da7 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,7 +4,6 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() - int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,14 +13,11 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - revisiontotal += importedmeta.buildstampinfo.revision + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders.add("${destFolder}/**") } } - def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) - return [artifactFolders, revisiontotal] + return artifactFolders } return this; diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..081226a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,6 @@ +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -7,7 +9,9 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +25,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ae7e5f4..e73782f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,4 +136,9 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + return newrevision +} + return this; \ No newline at end of file From e3bf9774ede490241fd0840e834e5230e4c9dc4d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 21:25:38 +0000 Subject: [PATCH 15/31] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- versionnumbering/generate.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e73782f..7e5568a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,6 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" return newrevision } From 2dbfbdab472fc59f27e233fb3f9fdf5759a99278 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:14:13 +0000 Subject: [PATCH 16/31] Remove .git files/folders before publishing --- publish/do.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..91f3b3e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepo}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs rm -rvf + find . -name ".gitignore" | xargs rm -rvf + find . -name ".gitmodules" | xargs rm -rvf + find . -iname ".gitattributes" | xargs rm -rvf """ } } From 6ac6396c6f7ffeac25f73b7c8344d561b81e859d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:29:40 +0000 Subject: [PATCH 17/31] Remove .git files/folders before publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 91f3b3e..ec9b53f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,7 +90,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { - echo "Stripping git history out of ${releaserepo}" + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash pwd From 7a4c46d342ac3671f42d2077be48d0bbf21176ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 14:01:09 +0000 Subject: [PATCH 18/31] Releasing to git was relying on the source code sha changing only; However it needs to take into account that a dependent component revision number changed too --- release/do.groovy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } From ca914eebd209abcefaca8ffbbd1f21847e9de156 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:20:25 +0000 Subject: [PATCH 19/31] Remove .git detritus from published folders --- publish/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index ec9b53f..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -94,10 +94,10 @@ def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash pwd - find . -type d -name ".git" | xargs rm -rvf - find . -name ".gitignore" | xargs rm -rvf - find . -name ".gitmodules" | xargs rm -rvf - find . -iname ".gitattributes" | xargs rm -rvf + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } From 5a1a55aa35a0b9d2a4fae58c9c3143947aa65e68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:22:15 +0000 Subject: [PATCH 20/31] Move product agnostic scripts into common subtree area --- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From fa3cc37a4d9ec6c79c55aaa09e2772aea9065977 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 16:47:07 +0000 Subject: [PATCH 21/31] Post subtree pull corrections --- publish/do.groovy | 7 +++- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/do.groovy | 14 ++++++- release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From e00d396c2b5dab238d2b0bff57abf0a1d0a0c082 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 17:49:02 +0000 Subject: [PATCH 22/31] Remove redundant common_scripts --- file/utils.groovy | 16 --- versionnumbering/versioninfo.py | 180 -------------------------------- 2 files changed, 196 deletions(-) delete mode 100644 file/utils.groovy delete mode 100644 versionnumbering/versioninfo.py diff --git a/file/utils.groovy b/file/utils.groovy deleted file mode 100644 index 0ffa47c..0000000 --- a/file/utils.groovy +++ /dev/null @@ -1,16 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - -// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents -def currentContents(file, newline = "${nl}", defaultContents = "") { - def contents = "" - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - -return this; diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py deleted file mode 100644 index 3d2a80d..0000000 --- a/versionnumbering/versioninfo.py +++ /dev/null @@ -1,180 +0,0 @@ -# encoding: utf-8 -''' -versionnumbering -- shortdesc - -versionnumbering generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess -import re -import datetime - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - thirdpartyversionprefix = "1.25000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().find("master") != -1: # Special case - return mastertypeversionprefix - - splits2 = re.split(r'(\d+)', sourcebranch) - splits = [] - hasdot = False - for split in splits2: - if split.isdigit(): - splits.append(split) - if split == ".": - hasdot = True - - if not hasdot: - print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) - return thirdpartyversionprefix - - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - sourcebranch = args.sourcebranch - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - - print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber) - - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) From 1eba3bf0162fead6284b53e63647c2f2456326d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 18:00:34 +0000 Subject: [PATCH 23/31] Remove compile retries --- compile/eclipsebuild/eclipseds5build.groovy | 52 ++++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..cf382b1 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,39 +142,35 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ } } - else { - echo "Not building ${build_variant} variant of project: ${project}" + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } - } catch (Exception e) { - echo "Build failed - Waiting for a while" - sleep(time: 4, unit: 'SECONDS') - error "Build failed..." } + else { + echo "Not building ${build_variant} variant of project: ${project}" + } + } catch (Exception e) { + error "Build failed..." } } } From 8ae90886986d45e2261d7a1674737241b85298a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:38:54 +0000 Subject: [PATCH 24/31] Fixing version stamping in windows --- versionnumbering/do.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 9b4a42e..0c53ce9 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -1,17 +1,20 @@ def stamp(convertdetails) { for (convertdetail in convertdetails) { - filetoconvert = convertdetail[0] - pattern = convertdetail[1] - replacement = convertdetail[2] + def filetoconvert = convertdetail[0] + def pattern = convertdetail[1] + def replacement = convertdetail[2] + echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" + echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ - "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } else { bat """ - "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + "%${PYTHON}%" -u "%${stampscript}%" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" """ } } From fc461ba62b9caf0e2f7d1c8215f7aa309ee45a4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:46:44 +0000 Subject: [PATCH 25/31] Fixing version stamping in windows --- versionnumbering/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 0c53ce9..9f45f70 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -14,7 +14,7 @@ def stamp(convertdetails) { } else { bat """ - "%${PYTHON}%" -u "%${stampscript}%" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" """ } } From 7bd8655a9224716056a3ec25beed4d4ba97ad6b3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 11:59:26 +0000 Subject: [PATCH 26/31] Fixing version stamping in windows --- versionnumbering/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy index 9f45f70..666aee3 100644 --- a/versionnumbering/do.groovy +++ b/versionnumbering/do.groovy @@ -4,9 +4,9 @@ def stamp(convertdetails) { def filetoconvert = convertdetail[0] def pattern = convertdetail[1] def replacement = convertdetail[2] - echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + //echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" - echo "Using python: ${PYTHON} and script: ${stampscript}" + //echo "Using python: ${PYTHON} and script: ${stampscript}" if (isUnix()) { sh """ "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" From 93ef7dfcf78b80b3eb033320caa9494a0052cb33 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 12:05:47 +0000 Subject: [PATCH 27/31] Revert "Remove compile retries" This reverts commit 90ee2b5c1c5c2bcf13af8d3c501114f41b3d8181. --- compile/eclipsebuild/eclipseds5build.groovy | 52 +++++++++++---------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index cf382b1..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,35 +142,39 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } + } catch (Exception e) { + echo "Build failed - Waiting for a while" + sleep(time: 4, unit: 'SECONDS') + error "Build failed..." } - else { - echo "Not building ${build_variant} variant of project: ${project}" - } - } catch (Exception e) { - error "Build failed..." } } } From 2371eff2e841e448c32bf7c2e84f5023a0b44e19 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 17:24:46 +0000 Subject: [PATCH 28/31] Move eclipse nios operation into common area --- compile/eclipsebuild/eclipseniosbuild.groovy | 103 +++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 compile/eclipsebuild/eclipseniosbuild.groovy diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy new file mode 100644 index 0000000..18c31d1 --- /dev/null +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -0,0 +1,103 @@ +// Gets the debug and optimization flags. +// Arg1 is "app" or "lib". Returns flags. +// Restore the debug and optimization levels, in case this is a dev machine. +// Arg1 is "app" or "lib". Arg2-3 is flags from getFlags. Returns nothing. +def setFlagsToRelease(NIOSEDS, project, wdir) { + if ( project == "app" ) { + if (isUnix()) { + sh """#!/bin/bash + set -x + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-app-update-makefile --app-dir=. --set-optimization=-O2 --set-debug-level= + """ + } + } + else if ( project == "lib" ) { + if (isUnix()) { + sh """#!/bin/bash + set -x + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-lib-update-makefile --lib-dir=. --set-optimization=-O2 --set-debug-level= + """ + } + } +} + +def remake_and_check(NIOSEDS, wdir) { + if (isUnix()) { + sh """#!/bin/bash + set -x + cd "${wdir}" + + # Use the makefile to clean and build + "${NIOSEDS}/nios2_command_shell.sh" make clean + if [ \$? -ne 0 ]; then exit 1; fi + + # Set the debug and optimization levels to production values + "${NIOSEDS}/nios2_command_shell.sh" make -j12 all + if [ \$? -ne 0 ]; then exit 1; fi + """ + } +} + +def generate_bsp(NIOSEDS, bspfolder) { + if (isUnix()) { + sh """#!/bin/bash + set -x + + "${NIOSEDS}/nios2_command_shell.sh" nios2-bsp-generate-files --bsp-dir "${bspfolder}" --log "${bspfolder}/bsp.log" --settings "${bspfolder}/settings.bsp" + """ + } +} + +def modifylib(NIOSEDS, bspfolder, buildconfig, wdir) { + if (isUnix()) { + sh """#!/bin/bash + set -x + + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-lib-update-makefile --lib-dir=. --set-bsp-dir "${bspfolder}" --set-active-build-config "${buildconfig}" + """ + } +} + +def modifyapp(NIOSEDS, bspfolder, wdir) { + if (isUnix()) { + sh """#!/bin/bash + set -x + + cd "${wdir}" + + "${NIOSEDS}/nios2_command_shell.sh" nios2-app-update-makefile --app-dir=. --set-bsp-dir "${bspfolder}" + """ + } +} + +def dump_elf_info(NIOSEDS, elffile, outputdir) { + sh """#!/bin/bash + set -x + + mkdir -p "${outputdir}" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name > "${outputdir}/lower_cpu_name.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp > "${outputdir}/lower_sidp.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id > "${outputdir}/lower_id.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp > "${outputdir}/lower_timestamp.txt" + if [ \$? -ne 0 ]; then exit 1; fi + + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo > "${outputdir}/lower_sopcinfo.txt" + if [ \$? -ne 0 ]; then exit 1; fi + """ +} + +return this; \ No newline at end of file From 5127186c4f974c68c635d82a28ef6d7a80be1ee9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 18:26:50 +0000 Subject: [PATCH 29/31] Output version information to meta --- compile/eclipsebuild/eclipseniosbuild.groovy | 55 ++++++++++++-------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy index 18c31d1..a3bb4b9 100644 --- a/compile/eclipsebuild/eclipseniosbuild.groovy +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -76,28 +76,39 @@ def modifyapp(NIOSEDS, bspfolder, wdir) { } } -def dump_elf_info(NIOSEDS, elffile, outputdir) { - sh """#!/bin/bash - set -x - - mkdir -p "${outputdir}" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name > "${outputdir}/lower_cpu_name.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp > "${outputdir}/lower_sidp.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id > "${outputdir}/lower_id.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp > "${outputdir}/lower_timestamp.txt" - if [ \$? -ne 0 ]; then exit 1; fi - - "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo > "${outputdir}/lower_sopcinfo.txt" - if [ \$? -ne 0 ]; then exit 1; fi - """ +def dump_elf_info(NIOSEDS, elffile, meta, custominfokeyprefix) { + if (isUnix()) { + def cpu_name = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_cpu_name", cpu_name) + def sidp = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_sidp", sidp) + def id = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_id", id) + def timestamp = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_timestamp", timestamp) + def sopcinfo = sh(script: """#!/bin/bash + `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo` + """, + returnStdout: true + ) + meta.addCustomInfo("${custominfokeyprefix}_sopcinfo", sopcinfo) + } } return this; \ No newline at end of file From a20fce505017824b40b7185b6ee4127e24a92c75 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 18:33:06 +0000 Subject: [PATCH 30/31] Output version information to meta --- compile/eclipsebuild/eclipseniosbuild.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy index a3bb4b9..ce7d798 100644 --- a/compile/eclipsebuild/eclipseniosbuild.groovy +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -79,31 +79,31 @@ def modifyapp(NIOSEDS, bspfolder, wdir) { def dump_elf_info(NIOSEDS, elffile, meta, custominfokeyprefix) { if (isUnix()) { def cpu_name = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_cpu_name", cpu_name) def sidp = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_sidp", sidp) def id = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_id", id) def timestamp = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp """, returnStdout: true ) meta.addCustomInfo("${custominfokeyprefix}_timestamp", timestamp) def sopcinfo = sh(script: """#!/bin/bash - `"${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo` + "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo """, returnStdout: true ) From d1f32dc580e9b5b299fddcc11d0289a780f00b00 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 21:33:59 +0000 Subject: [PATCH 31/31] Output version information to meta --- compile/eclipsebuild/eclipseniosbuild.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseniosbuild.groovy b/compile/eclipsebuild/eclipseniosbuild.groovy index ce7d798..a808932 100644 --- a/compile/eclipsebuild/eclipseniosbuild.groovy +++ b/compile/eclipsebuild/eclipseniosbuild.groovy @@ -82,32 +82,34 @@ def dump_elf_info(NIOSEDS, elffile, meta, custominfokeyprefix) { "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --cpu_name """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_cpu_name", cpu_name) def sidp = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sidp """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_sidp", sidp) def id = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --id """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_id", id) def timestamp = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --timestamp """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_timestamp", timestamp) +/* def sopcinfo = sh(script: """#!/bin/bash "${NIOSEDS}/nios2_command_shell.sh" nios2-elf-query "${elffile}" --sopcinfo """, returnStdout: true - ) + ).trim() meta.addCustomInfo("${custominfokeyprefix}_sopcinfo", sopcinfo) +*/ } }