From db6163fff766bbaecc039aa3ccafa9df7b2b0aee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Dec 2018 18:30:15 +0000 Subject: [PATCH 01/11] Add publishing script --- publish/do.groovy | 313 ++++++++++++++++++++++++++++++++++++++ versioncontrol/git.groovy | 18 ++- 2 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 publish/do.groovy diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..2c5a018 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,313 @@ +markunstable = false +unstablereason = "" + +FTP_CLIENT_LOCATION='ftp' +FTP_URL='ie1.hostedftp.com' +FTP_USER='ftp@globalinkjetsystems.com' +FTP_CUSTOMER_ROOT='/Customers/' +FTP_PASSWORD='ramshorn42' +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } +} + +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.meta + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" + def publishableitems = releasemeta.publishableinfo + def ArrayList publishablePaths = new ArrayList() + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean changed = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + return changed +} + +def ArrayList maketaglist(String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + taglist.add(othertagitem.trim()) + } + } + return taglist +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def publish(releasemeta, String uniquefolderforrepo) { + summaryappend(" Git operations: ") + def String buildstamp = releasemeta.buildstampinfo.buildstamp + + dir(uniquefolderforrepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + releasegitadd() + + def boolean changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamp) + } + else { + releasegitreset() + summaryappend("No-commit,") + } + + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } +} + +def recallcomponent(releaseinfo) { + def releasemeta = releaseinfo.meta + def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + + summaryappend("\n") +} + +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = releasemeta.releasewalkerunique(md) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + recallcomponent(releaseinfo) + } + // publish(releasemeta, uniquefolderforrepo) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index cb9354b..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,3 +1,7 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { @@ -6,7 +10,16 @@ String getGitInfo(String workingSetRootPath) { } } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +@NonCPS +def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return new SparseCheckoutPaths(sparsecheckoutpaths) +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRoot != "") { reporootpath = "${reporootenv.trim()}/${repoRoot}" @@ -28,7 +41,8 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + sparsecheckoutconfig(sparseitemlist) ], submoduleCfg: [], ]) From 9ea146b13318e6f5beedd7dd917d6cda79ac60b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 11:22:13 +0000 Subject: [PATCH 02/11] Only checkout repositories that have publishable items --- publish/do.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 2c5a018..c7084d0 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,11 +104,13 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" def publishableitems = releasemeta.publishableinfo def ArrayList publishablePaths = new ArrayList() - for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } def releasegitadd() { From a7f6f97d7720cebc81be274449a9634d3569cfbc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:15:10 +0000 Subject: [PATCH 03/11] Build up folder structure for FTP --- publish/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c7084d0..4d02b4f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -281,9 +281,9 @@ def publish(releasemeta, String uniquefolderforrepo) { } } -def recallcomponent(releaseinfo) { +def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -305,7 +305,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo) + recallcomponent(releaseinfo, customer, softwarerelease) } // publish(releasemeta, uniquefolderforrepo) From 67ca5024c206ccfd03e322717df27ae1458332f9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:59:24 +0000 Subject: [PATCH 04/11] Change root path for transfer; Add an example FTP operation --- publish/do.groovy | 53 ++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 4d02b4f..b953f4c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,11 +1,6 @@ markunstable = false unstablereason = "" -FTP_CLIENT_LOCATION='ftp' -FTP_URL='ie1.hostedftp.com' -FTP_USER='ftp@globalinkjetsystems.com' -FTP_CUSTOMER_ROOT='/Customers/' -FTP_PASSWORD='ramshorn42' summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" @@ -250,40 +245,32 @@ def sourcegitpush(String dryrunpush ) { } } -def publish(releasemeta, String uniquefolderforrepo) { - summaryappend(" Git operations: ") - def String buildstamp = releasemeta.buildstampinfo.buildstamp +def String FTP_CLIENT_LOCATION='ftp' +def String FTP_URL='ie1.hostedftp.com' +def String FTP_USER='ftp@globalinkjetsystems.com' +def String FTP_CUSTOMER_ROOT='/Customers/' +def String FTP_PASSWORD='ramshorn42' - dir(uniquefolderforrepo) { - def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Pre commit sha code: ${precommitshacode}" - - releasegitadd() +def publish() { + summaryappend(" FTP operations: ") - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamp) - } - else { - releasegitreset() - summaryappend("No-commit,") - } - - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" - - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - def tagadded = releasegittags(postcommitshacode, taglist) - - if (changed || tagadded) { - releasegitpush(dryrunpush) - } + dir(releaserepofolder) { + sh """#!/bin/bash + ftp -n -v ${FTP_URL} << EOT + ascii + user ${FTP_USER} ${FTP_PASSWORD} + prompt + cd ${FTP_CUSTOMER_ROOT} + ls -la + bye + EOT + """ } } def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -307,7 +294,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, customer, softwarerelease) } - // publish(releasemeta, uniquefolderforrepo) + publish() return [summarystring, markunstable, unstablereason] } From b178b86f6ee9199bdc418f5cf79d28b6505b7a00 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 14:26:20 +0000 Subject: [PATCH 05/11] Build up folder structure for FTP --- publish/do.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b953f4c..46b3fdd 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -245,15 +245,15 @@ def sourcegitpush(String dryrunpush ) { } } -def String FTP_CLIENT_LOCATION='ftp' -def String FTP_URL='ie1.hostedftp.com' -def String FTP_USER='ftp@globalinkjetsystems.com' -def String FTP_CUSTOMER_ROOT='/Customers/' -def String FTP_PASSWORD='ramshorn42' def publish() { summaryappend(" FTP operations: ") + def String FTP_CLIENT_LOCATION='ftp' + def String FTP_URL='ie1.hostedftp.com' + def String FTP_USER='ftp@globalinkjetsystems.com' + def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { sh """#!/bin/bash ftp -n -v ${FTP_URL} << EOT From 4230e4dcd33918892e2f4ac90fafb688dad4ba87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:14:29 +0000 Subject: [PATCH 06/11] Use ncftpput to transfer --- publish/do.groovy | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 46b3fdd..13a280a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -106,6 +106,9 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } + else { + summaryappend("None ") + } } def releasegitadd() { @@ -245,32 +248,26 @@ def sourcegitpush(String dryrunpush ) { } } - -def publish() { +def publish(String customer) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' def String FTP_URL='ie1.hostedftp.com' def String FTP_USER='ftp@globalinkjetsystems.com' - def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { + summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") sh """#!/bin/bash - ftp -n -v ${FTP_URL} << EOT - ascii - user ${FTP_USER} ${FTP_PASSWORD} - prompt - cd ${FTP_CUSTOMER_ROOT} - ls -la - bye - EOT + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ + summaryappend(", Transferred ") } } -def recallcomponent(releaseinfo, String customer, String softwarerelease) { +def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -292,9 +289,9 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo, customer, softwarerelease) + recallcomponent(releaseinfo, softwarerelease) } - publish() + publish(customer) return [summarystring, markunstable, unstablereason] } From e06d3705497981903880f07cb750004a800e5669 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:59:04 +0000 Subject: [PATCH 07/11] Use ncftpput to transfer --- publish/do.groovy | 200 +++------------------------------------------- 1 file changed, 10 insertions(+), 190 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 13a280a..8cdf9a3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -5,12 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -42,54 +37,7 @@ def summaryappend(String summary) { summarystring += summary } -def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = buildmeta.getNewMetaObject(this) - // See if there is meta data file from the current release - def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" - if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(checkoutfolder) - } - return previousreleasemeta -} - -def checkoutreleaserepo(buildmeta, String checkoutfolder) { - def String releaserepo = buildmeta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - def previousreleasemeta = buildmeta.getNewMetaObject(this) - dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") - } - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) - dir(checkoutfolder) { - // Clear all existing files - sh("git rm -r *") - } - return previousreleasemeta -} - -def checkoutsourcerepo(buildmeta, String checkoutfolder) { - def String sourcerepo = buildmeta.gitinfo.remoteurl - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } -} - +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -111,143 +59,6 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } } -def releasegitadd() { - // Add anything new - sh("git add .") - summaryappend("Add,") -} - -def releasegitreset() { - // Add anything new - sh("git reset --hard") - sh("git clean -fdx") - summaryappend("Reset,") -} - -def releasegitcommit(String buildstamp) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") -} - -// Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { - String buildsha = buildmeta.gitinfo.gitrevision - String previoussha = previousreleasemeta.gitinfo.gitrevision - - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return changed -} - -def ArrayList maketaglist(String buildstamp, String otherreleasetags) { - def ArrayList taglist = new ArrayList() - taglist.add(buildstamp) - - def ArrayList othertaglist = otherreleasetags.split(',') - for (othertagitem in othertaglist) { - if (othertagitem?.trim()) { - taglist.add(othertagitem.trim()) - } - } - return taglist -} - -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\"") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def boolean releasegittags(String postcommitshacode, ArrayList taglist) { - boolean tagadded = false - - summaryappend("Rel Tag ") - for (tag in taglist) { - if (releasegittag(postcommitshacode, tag)) { - tagadded = true - } - } - return tagadded -} - -def boolean sourcegittag(String shacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != shacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\" ${shacode}") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def sourcegittags(String buildsha, List taglist) { - boolean tagadded = false - - summaryappend(",Src Tag ") - for (tag in taglist) { - if (sourcegittag(buildsha, tag)) { - tagadded = true - } - } - return tagadded -} - -def releasegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend(",Dry-Run-Push") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend(",Push") - } -} - -def sourcegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags") - summaryappend(",Src-Dry-Run-Push") - setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags") - summaryappend(",Src Push") - } -} - def publish(String customer) { summaryappend(" FTP operations: ") @@ -278,6 +89,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { summaryappend("\n") } +def stripgithistory() { + dir(releaserepofolder) { + sh """#!/bin/bash + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + """ + } +} + def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() @@ -291,6 +110,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, softwarerelease) } + stripgithistory() publish(customer) return [summarystring, markunstable, unstablereason] From 8ba5a2c59a7407c8d993bd343793818953edbc7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 16:01:13 +0000 Subject: [PATCH 08/11] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 8cdf9a3..34d0e42 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -68,7 +68,7 @@ def publish(String customer) { def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { - summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") + summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ From 08e458d9d633bfca42d0dd39bb5e549066a6b277 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:12:45 +0000 Subject: [PATCH 09/11] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 34d0e42..a30bfc9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -92,7 +92,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf """ } } From 64a6b177de8bce34c2a37982ec5305bd62be2d96 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:53:08 +0000 Subject: [PATCH 10/11] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- artifacts/metaexport.groovy | 8 ++------ import/importer.groovy | 10 ++++++++-- versionnumbering/generate.groovy | 5 +++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b4d4a6d..0ca5da7 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,7 +4,6 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() - int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,14 +13,11 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - revisiontotal += importedmeta.buildstampinfo.revision + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders.add("${destFolder}/**") } } - def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) - return [artifactFolders, revisiontotal] + return artifactFolders } return this; diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..081226a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,6 @@ +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -7,7 +9,9 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +25,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ae7e5f4..e73782f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,4 +136,9 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + return newrevision +} + return this; \ No newline at end of file From e3bf9774ede490241fd0840e834e5230e4c9dc4d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 21:25:38 +0000 Subject: [PATCH 11/11] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- versionnumbering/generate.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e73782f..7e5568a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,6 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" return newrevision }