diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy new file mode 100644 index 0000000..043c4de --- /dev/null +++ b/artifacts/metaexport.groovy @@ -0,0 +1,24 @@ + +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(meta, subJobs) { + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + def ArrayList artifactFolders = new ArrayList() + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getProjectName() + def subJobNumber = subJobObject.getNumber() + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] + dir(destFolder) { + deleteDir() + } + importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) + artifactFolders.add("${destFolder}/**") + } + } + return artifactFolders +} + +return this; diff --git a/autobuilder.groovy b/autobuilder.groovy new file mode 100644 index 0000000..63ca2de --- /dev/null +++ b/autobuilder.groovy @@ -0,0 +1,88 @@ +// Assume that WORKSPACE environment variable is set + +autobuildroot = "${WORKSPACE}/autobuild" + +commonscriptsroot = "${autobuildroot}/Common_scripts" + +metafolderscript = null + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + stash allowEmpty: true, includes: 'autobuild/**', name: 'Clean workspace stash', useDefaultExcludes: false + cleanWs() + unstash name: 'Clean workspace stash' + } + else { + echo "Not cleaning workspace" + } +} + +def commonscripts() { + return commonscriptsroot +} + +def getautobuildpath() { + return autobuildroot +} + +def getmetafolderscript() { + return metafolderscript +} + +// Start up boilderplate +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = getmetafolderscript().metareset(script) + meta.productinfo.set(productname, subcomponentname, jirakey) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + + def rr = load("${commonscripts()}/release/registerrepo.groovy") + rr.registerrepo(meta, releaserepo, repoRootSuffix) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +// Start up boilderplate +def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = getmetafolderscript().metareset(script) + meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.groovy") +} + +// Everyone wants to know this +autobuilder = this + +return this; diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..34eed98 --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,86 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..7bfe2d1 --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,77 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + + // Walk the meta information to a max depth + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..c2503cd --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,14 @@ + +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(meta, checksummable) { + for (filepath in checksummable) { + def checksumset = meta.checksuminfo.addChecksumSet(filepath) + def files = findFiles(glob: filepath) + for (file in files) { + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) + } + } +} + +return this; \ No newline at end of file diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file diff --git a/commandline/commandline.gsh b/commandline/commandline.groovy similarity index 100% rename from commandline/commandline.gsh rename to commandline/commandline.groovy diff --git a/compile/eclipsebuild/disableparallelbuilds.groovy b/compile/eclipsebuild/disableparallelbuilds.groovy new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.groovy @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + + registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) +} + +return this diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh deleted file mode 100644 index b91e0a3..0000000 --- a/configuration/configuration.gsh +++ /dev/null @@ -1,37 +0,0 @@ -gh = load("${WORKSPACE}/Common_scripts/versioncontrol/git.gsh") - -configurationCollection = [:] - -def registerConfigurationInfo(String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - def ConfigurationInfo = [configurationURL, configurationVersion, configurationSHA] - configurationCollection.put(configurationName, ConfigurationInfo) -} - -def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${WORKSPACE}")) -} - -def saveConfigurationInVersionInfo() { - dir("${WORKSPACE}/version") { - echo "Writing configuration info to ${WORKSPACE}/version/configuration.txt" - writeFile file: 'configuration.txt', text: getFormattedConfiguration() - } -} - -String getFormattedConfiguration() { - String formattedInfo = "" - def keys = configurationCollection.keySet().sort() - for (key in keys) { - (configurationURL, configurationVersion, configurationSHA) = configurationCollection.get(key) - formattedLine = String.format("%s:%s,%s,%s%n", key, configurationURL, configurationVersion, configurationSHA) - formattedInfo += formattedLine - } - return formattedInfo -} - -def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { - gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(tool, gitURL, revision, gh.getGitInfo(targetDir)) -} - -return this diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..f71c083 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,62 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/logic", + "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/diff", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying logic artifacts') { + steps { + script { + env.LOGICDEST = "${WORKSPACE}/logic" + if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["embedded_logic/**"]) + sh """ + rm -v -rf diff meta embedded_logic + mv -v logic/* . + """ + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/differences/do.groovy b/differences/do.groovy new file mode 100644 index 0000000..32f18b5 --- /dev/null +++ b/differences/do.groovy @@ -0,0 +1,18 @@ + +def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { + def String cs = autobuilder.commonscripts() + def String difffile = meta.difffilepath() + def String jirakey = meta.productinfo.jirakey + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" + """ + } + else { + bat """ + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" + """ + } +} + +return this; diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy new file mode 100644 index 0000000..11b3733 --- /dev/null +++ b/differences/dometahierarchy.groovy @@ -0,0 +1,744 @@ +import groovy.json.JsonSlurper + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" + +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" + +reportfolder = "${WORKSPACE}/reports" + +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } +} + +class DifferenceRecord { + String productname + String subcomponentname + String jirakey + + String rawdifferencetext + String sha + String type + String mergeinfo + List jiraticketinfo + String comment + + DifferenceRecord() { + productname = "" + subcomponentname = "" + jirakey = "" + + rawdifferencetext = "" + sha = "" + type = "" + mergeinfo = "" + jiraticketinfo = [] + comment = "" + } + + def String dump() { + String jiraticketstring = '' + for (j in jiraticketinfo) { + jiraticketstring += j.dump() + " " + } + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + } +} + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } + dir(reportfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def showmetaitems(metaitems) { + String info = '' + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + def offsetpath = relatedmetainfo.offsetpath + info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" + } + echo info +} + +def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { + boolean checkedout = false + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo?.trim()) { + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } + checkedout = true + } + return checkedout +} + +def String getcomponent(meta) { + def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) + + return [checkedout, uniquefolderforsourcerepo] +} + +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def differencerecords = [] + def String releaseexpression + if (tripledot) { + releaseexpression = earlyrevision + "..." + laterrevision + } else { + releaseexpression = earlyrevision + ".." + laterrevision + } + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() + } + return differencetext +} + +def extractsha(String line) { + def pattern = ~"^(\\p{XDigit}*) (.*)" + def match = (line =~ pattern) + match.find() + + def sha = match.group(1) + def restofline = match.group(2) + + //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" + + return [sha.trim(), restofline.trim()] +} + +def extractmerges(String line) { + def pattern1 = "^(.*)(Merge branch .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + def pattern2 = "^(.*)(Merge remote-tracking branch .*)" + def match2 = (line =~ pattern2) + if (match2.find()) { + return [true, match2.group(2)] + } + + def pattern3 = "^(.*)(Merge commit .*)" + def match3 = (line =~ pattern3) + if (match3.find()) { + return [true, match3.group(2)] + } + + def pattern4 = "^(.*)(Add .* from commit .*)" + def match4 = (line =~ pattern4) + if (match4.find()) { + return [true, match4.group(2)] + } + + return [false, line] +} + +def jiraticketexists(String jiraticket) { + def String URI = "${JIRA_URI}/browse/${jiraticket}" + def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" + def get = new URL(RestfulURI).openConnection(); + get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) + get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) + get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) + get.setRequestProperty( 'Accept', 'application/json' ) + def getRC = get.getResponseCode(); + + def String redirectedjiraticket = jiraticket + if(!getRC.equals(200)) { + return [false, '', jiraticket] + } else { + def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) + redirectedjiraticket = jsonresponse['key'] + } + return [true, URI, redirectedjiraticket] +} + +def extractjiraticketinfo(String line) { + def List jiratickets = [] + boolean exists = false + String URI = '' + + while (true) { + line = line.trim() + def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line + def commamatch = (line =~ commapattern) + if (commamatch.find()) { + line = commamatch.group(1).trim() + } + def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def match1 = (line =~ pattern1) + if (match1.find()) { + jiraticket = match1.group(1) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match1.group(2) + } else { + break + } + } else { + def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 + def match2 = (line =~ pattern2) + if (match2.find()) { + jiraticket = match2.group(1) + "-" + match2.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match2.group(3) + } else { + break + } + } else { + def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 + def match3 = (line =~ pattern3) + if (match3.find()) { + jiraticket = match3.group(1) + "-" + match3.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match3.group(3) + } else { + break + } + } else { + break + } + } + } + } + boolean result = jiratickets.size() > 0 + return [result, jiratickets, line] +} + +def extractreverts(String line) { + boolean result = true + + def pattern1 = '^(.*)Revert \"(.*)\"' + def match1 = (line =~ pattern1) + if (match1.find()) { + def line2 = match1.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + def pattern2 = '^(.*)Partial Revert \"(.*)\"' + def match2 = (line =~ pattern2) + if (match2.find()) { + def line2 = match2.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + return [false, [], line] +} + +def DifferenceRecord fillrecord(meta, line) { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = line + + // Example of raw text to process: + // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py + + (sha, line2) = extractsha(line) + dr.sha = sha + + (result, mergeinfo) = extractmerges(line2) + if (result) { + dr.type = "Merge" + dr.mergeinfo = mergeinfo + } else { + (result, jiraticketinfo, comment) = extractreverts(line2) + if (result) { + dr.type = "Revert" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.comment = line2 + } + } + } + + return dr +} + +def List processtext(meta, String text) { + def List differencerecords = [] + + text.split('\n').each { + def line = it + if (line?.trim()) { + def DifferenceRecord dr = fillrecord(meta, line.trim()) + differencerecords.add(dr) + //echo "${dr.dump()}" + } + } + return differencerecords +} + +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + differencerecords = processtext(earlymeta, differencetext) + if (earlyrevision == laterrevision) { + summaryappend(" No change") + } + } else { + summaryappend("The early and later git urls don't match") + echo "The early and later git urls don't match" + result = false; + setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") + } + + return [result, differencerecords] +} + +def deletedcomponent(earlymetaitems, key) { + echo "Component ${key} has been deleted in later version" + summaryappend("Deleted ") +} + +def addedcomponent(latermetaitems, key) { + echo "Component ${key} has been added in later version" + summaryappend("Added ") +} + +class MergeReportRecord { + String comment + int duplicates + + MergeReportRecord() { + comment = '' + duplicates = 0 + } +} + +def mergereport(String reportsubfolder, LinkedHashMap merges, boolean iscomponent = true) { + dir(reportsubfolder) { + def String mergelist = "Merge list:\n" + def mergessorted = merges.sort() + for (amerge in mergessorted) { + def MergeReportRecord m = amerge.value + mergelist += "${m.comment} (${m.duplicates})\n" + } + writeFile(file: "mergelist.txt", text: mergelist) + } +} + +class CommentsRecord { + String comment + int duplicates + + CommentsRecord() { + comment = '' + duplicates = 0 + } +} + +class RevertsReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + RevertsReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def revertsreport(String reportsubfolder, LinkedHashMap reverts, boolean iscomponent = true) { + dir(reportsubfolder) { + def String revertlist = "Revert list:\n" + def revertssorted = reverts.sort() + for (arevert in revertssorted) { + def RevertsReportRecord r = arevert.value + revertlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + revertlist += "${c.comment} (${c.duplicates})\n" + } + revertlist += "\n" + } + writeFile(file: "revertlist.txt", text: revertlist) + } +} + +class TicketedReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + TicketedReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def ticketedreport(String reportsubfolder, LinkedHashMap ticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String ticketedlist = "Ticketed list:\n" + def ticketedsorted = ticketed.sort() + for (aticketed in ticketedsorted) { + def TicketedReportRecord r = aticketed.value + ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + ticketedlist += "${c.comment} (${c.duplicates})\n" + } + ticketedlist += "\n" + } + writeFile(file: "ticketedlist.txt", text: ticketedlist) + } +} + +class UnticketedReportRecord { + String comment + int duplicates + + UnticketedReportRecord() { + comment = '' + duplicates = 0 + } +} + +def unticketedreport(String reportsubfolder, LinkedHashMap unticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String unticketedlist = "Unticketed list:\n" + def unticketedsorted = unticketed.sort() + for (anunticketed in unticketedsorted) { + UnticketedReportRecord u = anunticketed.value + unticketedlist += "${u.comment} (${u.duplicates})\n" + } + writeFile(file: "unticketedlist.txt", text: unticketedlist) + } +} + +class TicketsReportRecord { + String jiraticket + String ticketURI + int duplicates + + TicketsReportRecord() { + jiraticket = '' + ticketURI = '' + duplicates = 0 + } +} + +def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { + dir(reportsubfolder) { + String ticketlist = "Ticket list:\n" + ticketssorted = tickets.sort() + for (ticketnumber in ticketssorted) { + TicketsReportRecord t = ticketnumber.value + ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" + } + writeFile(file: "ticketlist.txt", text: ticketlist) + } +} + +def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { + dir(reportsubfolder) { + String recordlist = "" + for (dr in differencerecords) { + recordlist += dr.dump() + "\n\n" + } + writeFile(file: "recordlist.txt", text: recordlist) + } +} + +def boolean report(meta, differencerecords, boolean iscomponent = true) { + def Map ticketed = [:] + def Map merges = [:] + def Map reverts = [:] + def Map unticketed = [:] + def Map tickets = [:] + + for (dr in differencerecords) { + if (dr.type == "Merge") { + def MergeReportRecord m + if (!merges.containsKey(dr.mergeinfo)) { + m = new MergeReportRecord() + m.comment = dr.mergeinfo + m.duplicates = 1 + } else { + m = merges[dr.mergeinfo] + m.duplicates += 1 + } + merges[dr.mergeinfo] = m + } + if (dr.type == "Revert") { + def RevertsReportRecord r + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!reverts.containsKey(j.jiranumber)) { + r = new RevertsReportRecord() + r.jiraticket = j.jiranumber + r.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + r = reverts[j.jiranumber] + def CommentsRecord c + if (!r.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + c = r.commentscollection[dr.comment] + c.duplicates += 1 + } + r.commentscollection[dr.comment] = c + } + reverts[j.jiranumber] = r + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 + } else { + t = tickets[j.jiranumber] + t.duplicates += 1 + } + tickets[j.jiranumber] = t + } + } + if (dr.type == "Ticketed") { + def TicketedReportRecord t + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!ticketed.containsKey(j.jiranumber)) { + t = new TicketedReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + t = ticketed[j.jiranumber] + def CommentsRecord c + if (!t.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + c = t.commentscollection[dr.comment] + c.duplicates += 1 + } + t.commentscollection[dr.comment] = c + } + ticketed[j.jiranumber] = t + def TicketsReportRecord ti + if (!tickets.containsKey(j.jiranumber)) { + ti = new TicketsReportRecord() + ti.jiraticket = j.jiranumber + ti.ticketURI = j.uri + ti.duplicates = 1 + } else { + ti = tickets[j.jiranumber] + ti.duplicates += 1 + } + tickets[j.jiranumber] = ti + } + } + if (dr.type == "Unticketed") { + UnticketedReportRecord u + if (!unticketed.containsKey(dr.comment)) { + u = new UnticketedReportRecord() + u.comment = dr.comment + u.duplicates = 1 + } else { + u = unticketed[dr.comment] + u.duplicates += 1 + } + unticketed[dr.comment] = u + } + } + String reportsubfolder = "" + if (iscomponent) { + reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" + } else { + reportsubfolder = "${reportfolder}/Combined" + } + mergereport(reportsubfolder, merges, iscomponent) + revertsreport(reportsubfolder, reverts, iscomponent) + ticketedreport(reportsubfolder, ticketed, iscomponent) + unticketedreport(reportsubfolder, unticketed, iscomponent) + ticketsreport(reportsubfolder, tickets, iscomponent) + differencerecordsreport(reportsubfolder, differencerecords, iscomponent) +} + +def compare(earlymetaitems, latermetaitems, key) { + boolean result = true; + echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].meta + def latermeta = latermetaitems[key].meta + + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + + def boolean repoavailable + def String uniquefolderforsourcerepo + (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) + + List differencerecords = [] + if (repoavailable) { + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + summaryappend(" Items: ${differencerecords.size()} ") + report(earlymeta, differencerecords) + } else { + summaryappend(" No source repo ") + } + summaryappend("\n") + + return [result, differencerecords]; +} + +def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def List differencerecords = [] + + def int md = maxdepth.toInteger() + resetfolders() + preconditions() + + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + + def earlymeta = meta.importrelated(earlymetafolder) + def latermeta = meta.importrelated(latermetafolder) + + def earlymetaitems = earlymeta.releasewalkerunique(md) + def latermetaitems = latermeta.releasewalkerunique(md) + + echo "Early items:" + showmetaitems(earlymetaitems) + + echo "Later items:" + showmetaitems(latermetaitems) + + for (key in earlymetaitems.keySet()) { + if (latermetaitems.containsKey(key)) { + (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + differencerecords = differencerecords + componentdifferencerecords + } else { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + deletedcomponent(earlymetaitems, key) + summaryappend("\n") + } + } + for (key in latermetaitems.keySet()) { + if (!earlymetaitems.containsKey(key)) { + summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + addedcomponent(latermetaitems, key) + summaryappend("\n") + } + } + report(meta, differencerecords, false) + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..ab4102f --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,16 @@ +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { + if (isUnix()) { + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } + sh """ + diff -c ${wsparameter} "${importedfile}" "${currentfile}" + """ + } + else { + error "Windows not currently supporting file comparison" + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy new file mode 100644 index 0000000..5ac37f2 --- /dev/null +++ b/differences/json.groovy @@ -0,0 +1,15 @@ +def comparefiles(importedjsonfile, currentjsonfile) { + if (isUnix()) { + sh """ + which jq + if [ \$? -ne 0 ]; then exit 1; fi + # jq -S sorts json output + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") + """ + } + else { + error "Windows not currently supporting json file comparison" + } +} + +return this; diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy new file mode 100644 index 0000000..4bcdcdc --- /dev/null +++ b/differences/metahierarchy.groovy @@ -0,0 +1,91 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + SUMMARYDIR = "${WORKSPACE}/summary" + REPORTDIR = "${WORKSPACE}/reports" + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + "${EARLYDIR}", + "${LATERDIR}", + "${SUMMARYDIR}", + "${REPORTDIR}", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Obtain top level meta') { + steps { + script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() + importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') + importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') + } + } + } + stage('Diff components') { + steps { + script { + def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy") + (summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,reports/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index a778892..5c0f15d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,10 +29,6 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(versionfolder): - print("Creating folder (if not already there): {0}".format(versionfolder)) - os.makedirs(versionfolder, exist_ok = True) - def printDivider(outfh, times = 1, length = 100): for _ in range(times): outfh.write("{0}\n".format("*" * length)) @@ -72,6 +68,21 @@ jiraprojectpattern = { 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), + 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), + ), + 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), + ), + 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") @@ -160,7 +171,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -171,14 +182,14 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - versionfolder = args.versionfolder + difffile = args.difffile print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - if versionfolder: - makeFolder(versionfolder) - outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + if difffile: + os.makedirs(os.path.dirname(difffile), exist_ok=True) + outfh = open(difffile, 'w') else: outfh = sys.stdout @@ -197,7 +208,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if versionfolder: + if difffile: outfh.close() return 0 except Exception as e: diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..ba5eb0b --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,121 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + excludes: excludes, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: specific(buildnumber.toString()), + target: destpath + ) + def importedmeta = meta.importrelated(destpath) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + excludes: excludes, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: buildselector + ], + target: destpath + ) + def importedmeta = meta.importrelated(destpath) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def rewritemetafileasreleased(meta, String filepath) { + // Modify the meta info to say that is has already been released + meta.markreleased() + meta.saveJsonSerialised(filepath) +} + +publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(destpath) + + relatedimportinfo = importedmeta.getassociated() + echo "Cloning associated repos" + for(relatedmetainfo in relatedimportinfo) { + def relatedmeta = relatedmetainfo.meta + def relatedoffsetpath = relatedmetainfo.offsetpath + def cloneurl = relatedmeta.releaseinfo.releaserepoURL + def clonerevision = "refs/tags/${relatedmeta.releasetag()}" + def clonepath = "${destpath}/${relatedoffsetpath}" + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + + gh.stripgithistory(clonepath) + + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(importedmeta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(destpath) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(importedmeta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { + switch (source) { + case "Jenkins job artifact": + return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) + case "Release repository": + if (nested) { + return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + +return this; diff --git a/install/installhelper.groovy b/install/installhelper.groovy new file mode 100644 index 0000000..6fcdc6c --- /dev/null +++ b/install/installhelper.groovy @@ -0,0 +1,112 @@ +import hudson.FilePath + +// Transfer file from intermediate position to final position; +// creating necessary folders on the way; +// deleting intermediate file +def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) { + def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel() + + def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}" + def String abs_export_out = "${abs_exports_root_path}/${finalFilename}" + def export_filepath = new hudson.FilePath(channel, abs_export_out) + def export_out_folder = export_filepath.getParent() + def export_out_folder_string = export_out_folder.toString() + def export_out_file = export_filepath.getName() + + echo "Creating folder ${export_out_folder_string}" + export_out_folder.mkdirs() + + echo "Copying from ${pathToIntermediate} to ${abs_export_out}" + def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate) + intermediate_filepath.copyTo(export_filepath) + intermediate_filepath.delete() +} + +// Given details about artifacts to import and details of installers to build, import artifacts that are required +def Map artifactsImporter(meta, artifactDetails, installerDetails) { + artifactFetched = [:] + + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + for (installer in toBuild) { + def artifactKeys = installerDetails.get(installer)[1] + if (artifactKeys instanceof java.lang.String) { + artifactKeys = [ artifactKeys ] + } + for (artifactKey in artifactKeys) { + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + if (artifactFetched.get(artifactKey) == null) { + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) + artifactFetched.put(artifactKey, importmeta) + } + } + } + return artifactFetched +} + +@NonCPS +def checkISMfilebackend(String xml) { + def String message = "" + + def parsedxml = new XmlParser(true, true, true).parseText(xml) + def pathvariableinfo = [:] + def exitcode = 0 + // Load up a map of path variable and path pairs + // Ignore blank path definitions + // Ignore the workspace definition which should have the only absolute path + parsedxml.each{ table -> + if (table.@name == "ISPathVariable") { + table['row'].each{ row -> + def pathname = row['td'][0].value()[0] + def path = row['td'][1].value()[0] + if (path && pathname != 'WORKSPACE') { + pathvariableinfo[pathname] = path + } + } + } + } + // Check that each path variable is defined beginning with a path variable or the WORKSPACE one + pathvariableinfo.each{ pair -> + message += "pair key: ${pair.key} value: ${pair.value}\n" + def path = pair.value + def pathvarmatcher = path =~ /<(.*)>.*/ + if (pathvarmatcher.matches()) { + def pathvarfound = pathvarmatcher.group(1) + if (pathvarfound != 'WORKSPACE') { + if (!pathvariableinfo.containsKey(pathvarfound)) { + message += "Undefined path variable ${pathvarfound} found\n" + exitcode = 1 + } + } + } else { + message += "Path: ${path} does not begin with a path variable\n" + exitcode = 1 + } + } + return [exitcode, message] +} + +def checkISMfile(String ISMfile) { + echo "Checking ISM file: ${ISMfile}" + def xml = readFile(ISMfile) + (exitcode, message) = checkISMfilebackend(xml) + echo message + if (exitcode > 0) { + error "ISM file errors found" + } else { + echo "Checked ISM file - ok" + } +} + +certificatethumbprints = [:] +certificatethumbprints['floreana'] = '0FDEB0A227982C877146CA595091173AC9655479' // Authoring takes place on this machine - so this is what should be in the ISM file already +certificatethumbprints['hawaii'] = '49FD1B361CFF35014B81940C5B046543045DD34B' + +def selectSigningCertificate(String nodeName, String ISMfile) { + def masterpattern = '0FDEB0A227982C877146CA595091173AC9655479' + echo "Substituting occurrences of ${masterpattern} with ${certificatethumbprints[nodeName]} in ISM file: ${ISMfile}" + + textutils_subst = load("${autobuilder.commonscripts()}/textutils/subst.groovy") + textutils_subst.substitute(ISMfile, masterpattern, certificatethumbprints[nodeName]) +} + +return this; diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..ddc99c4 --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,795 @@ +import java.nio.file.Paths +import java.nio.file.Path +import groovy.json.JsonSlurperClassic + +abstract class GISJsonSerialised implements Serializable { + def script + + GISJsonSerialised(script) { + this.script = script + } + + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + String productname + String subcomponentname + String jirakey + + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + + void set(String productname, String subcomponentname, String jirakey) { + // Make sure the fields can be used in git tags + def int pstatus = 0 + if (script.isUnix()) { + pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + pstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${productname}\" > NUL 2> NUL", , returnStatus: true) + } + if (pstatus > 0) { + script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") + } + def int sstatus = 0 + if (script.isUnix()) { + sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) + } + else { + sstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${subcomponentname}\" > NUL 2> NUL", , returnStatus: true) + } + if (sstatus > 0) { + script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") + } + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + + void deserialiseJson(rawmeta) { + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + jirakey = rawmeta['jirakey'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + super(script) + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + void deserialiseJson(rawmeta) { + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + String buildjobname + int buildnumber + private String buildmachine + private String builddate + + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { + super(script) + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + void deserialiseJson(rawmeta) { + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + String buildstampprefix + int revision + String buildstamp + + BuildStampInfo(script, buildstampprefix = "1.30000.0.", revision = 0, buildstamp = "1.30000.0.0") { + super(script) + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + } + + void set(String buildstampprefix, int revision, String buildstamp) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + } + + String buildstampcommasep() { + def String buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + + void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") + revision = newrevision + def List buildstampsplit = buildstamp.split('\\.') + buildstampsplit[-1] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + script.echo("Build stamp now ${buildstamp}") + } + + void set(String buildstampprefix, int revision) { + buildstampprefix += buildstampprefix.endsWith('.') ? '' : '.' + this.buildstampprefix = buildstampprefix + this.buildstamp = buildstampprefix + "0" + changerevision(revision) + } + + void set(String buildstampprefix, String revision) { + set(buildstampprefix, revision.toInteger()) + } + + void deserialiseJson(rawmeta) { + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + String releaserepoURL + ArrayList releaseitems + boolean released + + ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { + super(script) + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() + this.released = released + } + + void set(String releaserepoURL = "") { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + script.echo("Adding release item: ${releaseitem} to release items") + releaseitems.add(releaseitem) + } + + void markreleased() { + this.released = true + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items + } + + void deserialiseJson(rawmeta) { + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + released = rawmeta['released'] ?: false + } + + LinkedHashMap serialisationdata() { + def data = [ + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), + 'released' : released + ] + return data + } +} + +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath = "", String sum = "") { + super(script) + this.filepath = filepath + this.sum = sum + } + + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String artifactpath + private ArrayList checksumpairs + + ChecksumSet(script, artifactpath = "") { + super(script) + this.artifactpath = artifactpath + checksumpairs = new ArrayList() + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } +} + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + checksumsets = new ArrayList() + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + +class PublishableInfo extends GISJsonSerialised { + private String filepath + + PublishableInfo(script, String filepath = "") { + super(script) + this.filepath = filepath + } + + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private static String FORMAT = "0.1" + private static String METAFOLDERNAME = "meta" + private static String DIFFFOLDERNAME = "diff" + private static String METAFILENAME = "info.json" + private static String METADIFFFILENAME = "differences.txt" + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String DIFFARTIFACTS = "${DIFFFOLDERNAME}/**" + + class RelatedMetaInfo extends GISJsonSerialised { + String offsetpath + MetaInfo meta + + RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + super(script) + this.offsetpath = offsetpath + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'offsetpath' : offsetpath, + 'meta' : meta.serialisationdata(), + ] + return data + } + } + + private String format = FORMAT + ProductInfo productinfo + GitInfo gitinfo + JenkinsBuildInfo jenkinsbuildinfo + BuildStampInfo buildstampinfo + ReleaseInfo releaseinfo + ChecksumInfo checksuminfo + private ArrayList configurationinfo + private ArrayList publishableinfo + private LinkedHashMap custominfo + private ArrayList associatedmeta + + MetaInfo(script) { + super(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) + configurationinfo = new ArrayList() + publishableinfo = new ArrayList() + custominfo = new LinkedHashMap() + associatedmeta = new ArrayList() + } + + List getassociated() { + return associatedmeta + } + + void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") + releaseinfo.markreleased() + for (associated in associatedmeta) { + associated.meta.releaseinfo.markreleased() + } + } + + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return configuration + } + + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") + publishableinfo.add(pi) + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + configurationinfo.add(ci) + } + + void addCustomInfo(String customId, String customValue) { + custominfo.put(customId.trim(), customValue) + } + + String getCustomInfo(String customId) { + return custominfo.getAt(customId.trim()) + } + + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(metaitem.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + if (rawmeta['format'] != FORMAT) { + script.error("Unsupported meta format found at ${filepath}") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + custominfo = rawmeta['custominfo'] + for (rawrelatedmeta in rawmeta['associatedmeta']) { + def rm = new RelatedMetaInfo(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + + def deserialiseJson(String filepath) { + script.echo("Deserialising JSON at file: ${filepath}") + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + def deserialiseJsonSlurper(String filepath) { + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) + deserialiseJson(rawmeta) + } + + String relativize(String mypath) { + String pathRelative + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + } else { + pathRelative = mypath + script.echo("Path relative is unchanged: ${pathRelative}") + } + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') + // Strip leading / if any + pathRelative = pathRelative.replaceAll('^/+', '') + + script.echo("Path relative is ${pathRelative}") + + return pathRelative + } + + static String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + + static String difffilepath() { + return DIFFFOLDERNAME + "/" + METADIFFFILENAME + } + + MetaInfo importrelated(String destpath) { + MetaInfo mi = new MetaInfo(script) + + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + metafilepath() + if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") + } + + RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) + associatedmeta.add(rmi) + + def importedmetadifffile = adjustedpath + "/" + difffilepath() + if (script.fileExists(importedmetadifffile)) { + def String newcontents = script.readFile(file: importedmetadifffile) + def String currentcontents = "" + if (script.fileExists(difffilepath())) { + currentcontents = script.readFile(file: difffilepath()) + } + + script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) + } + + return mi + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), + 'custominfo' : custominfo, + 'associatedmeta' : associatedmetaserialised(), + ] + return data + } + + static String folder() { + return METAFOLDERNAME + } + + static String difffolder() { + return DIFFFOLDERNAME + } + + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here + ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = "${METAARTIFACTS},${DIFFARTIFACTS}" + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } + saveJsonSerialised(metafilepath) + return artifactlist + } + + // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java + private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { + if (maxdepth > 0) { + if (releaseinfo.releaserepoURL) { + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + releasemetacollection.add(rmi) + } + if (pathsofar) { + pathsofar += "/" + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + relatedmetainfo.offsetpath + releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) + } + } + return releasemetacollection + } + + ArrayList releasewalker(int maxdepth, String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + return releasemetacollection + } + + String componentidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + } + + String componentversionuniqueidentifier() { + return componentidentifier() + "_" + buildstampinfo.buildstamp + } + + String globalreleasetag() { + return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber + } + + String releasetag() { + return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber + } + + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { + releasemetacollection.put(uniquestring, relatedmetainfo) + } + } else { + releasemetacollection.put(uniquestring, relatedmetainfo) + } + return releasemetacollection + } + + Map releasewalkerunique(int maxdepth, String startpath = "") { + def ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + + def Map releasemetauniquecollection = new LinkedHashMap() + for (relatedmetainfo in releasemetacollection) { + def String uniquestring = relatedmetainfo.meta.componentidentifier() + releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) + } + + return releasemetauniquecollection + } + + Map releasewalkerversionunique(int maxdepth, String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + + def Map releasemetaversionuniquecollection = new LinkedHashMap() + for (relatedmetainfo in releasemetacollection) { + def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() + releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) + } + + return releasemetaversionuniquecollection + } + + MetaInfo getNewMetaObject(script) { + return new MetaInfo(script) + } +} + +MetaInfo metareset(script) { + echo "Resetting meta folder" + def mi = new MetaInfo(script) + + dir(mi.folder()) { + deleteDir() + } + + dir(mi.difffolder()) { + deleteDir() + } + + return mi +} + +MetaInfo metafromfolder(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + +MetaInfo metafromfolderSlurper(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + +return this; diff --git a/platform/lineendings.groovy b/platform/lineendings.groovy new file mode 100644 index 0000000..9657b0d --- /dev/null +++ b/platform/lineendings.groovy @@ -0,0 +1,5 @@ + +// Put ${nl} into strings when wanting a newline +nl = System.properties['line.separator'] + +return this; diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..18ec404 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,177 @@ +import groovy.io.FileType + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(filteredreleaserepofolder) { + deleteDir() + } +} + +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${productuniquefolder(releasemeta)}" +} + +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def guessreporootsuffix(componentURL) { + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + return guess +} + +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") +def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.meta + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" + def publishableitems = releasemeta.publishableinfo + + def ArrayList publishablepaths = new ArrayList() + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + } + } + if (publishablepaths.size() > 0) { + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone + } + else { + summaryappend("None ") + } +} + +def publish(String customer, String finalreleaserepofolder = releaserepofolder) { + summaryappend(" FTP operations: ") + + def String FTP_CLIENT_LOCATION='ftp' + def String FTP_URL='ie1.hostedftp.com' + def String FTP_USER='ftp@globalinkjetsystems.com' + def String FTP_CUSTOMER_ROOT="/Customers/${customer}" + def String FTP_PASSWORD='ramshorn42' + dir(finalreleaserepofolder) { + summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") + sh """#!/bin/bash + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . + """ + summaryappend(", Transferred ") + } +} + +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdirs(); + } + dirFrom.eachFile(FileType.FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { + def releasemeta = releaseinfo.meta + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) + + summaryappend("\n") +} + +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for artifacts to publish + def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) + + echo "Version unique components:" + def String versionuniquecomponent = "" + for (key in componentReleaseInfoVersionUnique.keySet()) { + versionuniquecomponent += "${key}\n" + } + echo versionuniquecomponent + + def itempathstopublish = convertitemstopublish(itemstopublishstring) + for (key in componentReleaseInfoVersionUnique.keySet()) { + def releaseinfo = componentReleaseInfoVersionUnique[key] + recallcomponent(releaseinfo, softwarerelease, itempathstopublish) + } + gh.stripgithistory(releaserepofolder) + publish(customer) + + return [summarystring, markunstable, unstablereason] +} + +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(releasemeta, "", customer, softwarerelease, maxdepth) +} + +return this; diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..b478a5d --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,89 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy new file mode 100755 index 0000000..99e424f --- /dev/null +++ b/publish/publish_item_select.groovy @@ -0,0 +1,135 @@ +#!/usr/local/bin/groovy + +jobnameext = jobname ?: "jobunique" +metarepoext = metarepo ?: "git@hydra:print-server-releases/print_server_whole.git" +outputselectext = outputselect ?: "description" +metalevelsstringext = metalevels ?: "2" + +//jobnameext = "jobunique" +//metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +//outputselectext = "description" +//metalevelsstringext = "2" + +metalevelsext = metalevelsstringext.toInteger() +dolog = false + +commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" + +def log(message) { + if (dolog) { + println message + } +} + +def createfolder(String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + log("Creating directory: ${folder}") + def s = ['mkdir', '-p', folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def gitCloneOrPull(String repo, String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + if (!(new File(folder).exists())) { + log("Cloning: ${repo} to: ${folder}") + createfolder(folder) + def s = ['git', 'clone', repo, folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } else { + log("Pull in: ${folder}") + def s = ['git', 'pull'].execute(null, new File(folder)) + s.consumeProcessOutput(sout, serr) + s.waitFor() + } + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def loadmetaclass(String commonscripts, String metafolder) { + def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" + + GroovyShell shell = new GroovyShell() + def metafolderscript = shell.parse(new File(metafolderscriptfile)) + def meta = metafolderscript.metafromfolderSlurper(metafolder) + return meta +} + +def makeworking() { + def r = ['whoami'].execute() + r.waitFor() + def user = r.text.trim() + + def rootfolder = "/home/${user}/tmp" + createfolder(rootfolder) + + def commonscripts = "${rootfolder}/commonscripts" + gitCloneOrPull(commonscriptsrepo, commonscripts) + + // Clone or pull the relevant meta repo + def metareporoot = "${rootfolder}/${jobnameext}" + gitCloneOrPull(metarepoext, metareporoot) + def metafolder = "${metareporoot}/meta" + + // Load the meta folder class that interprets the meta repo + def meta = loadmetaclass(commonscripts, metareporoot) + return meta +} + +def String outputDescription(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } +} + +def String outputSelection(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } +} + +def List getItems(meta) { + def List menuitems = new ArrayList() + def metaitems = meta.releasewalkerunique(metalevelsext) + log("metaitems is ${metaitems.getClass()}") + + try { + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + if (outputselectext == "description") { + outputDescription(menuitems, releaseinfo) + } + if (outputselectext == "selection") { + outputSelection(menuitems, releaseinfo) + } + } + return menuitems + } catch (Exception e) { + menuitems.add("Error ${e}") + return menuitems + } +} + +def meta = makeworking() +def items = getItems(meta) + +count = 0 +for (item in items) { + count += 1 + log ("${count}: ${item}") +} + +return items \ No newline at end of file diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..2daf6f6 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,27 @@ + + +// The filepath must end in "/**" or "/" - Glob wildcard - as a git sparse checkout +def allowed(String filepath) { + if (filepath.endsWith('/**') || filepath.endsWith('/')) { + def files = findFiles glob: filepath + if (files.length > 0) { + return + } else { + error "There are no files/folders under the publishable item: ${filepath}" + } + } + error "The publishable item: ${filepath} does not end in /** or / (for git sparse checkouts)" +} + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(meta, publishable) { + filelist = [] + for (filepath in publishable) { + if (filepath != meta.METAARTIFACTS) { + allowed(filepath) + } + meta.addPublishable(filepath) + } +} + +return this; diff --git a/release/do.groovy b/release/do.groovy new file mode 100644 index 0000000..1b21bcf --- /dev/null +++ b/release/do.groovy @@ -0,0 +1,421 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files - Note that this does not remove .git* + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } +} + +def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + def boolean result = true + for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { + def filterpath = artifactfilter + if (buildinfo.offsetpath) { + filterpath = buildinfo.offsetpath + '/' + artifactfilter + } + // Copy relevant artifacts to temp folder + echo "Copying artifacts: ${filterpath} to ${tempfolder}" + try { + copyArtifacts( + filter: filterpath, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + ], + target: tempfolder + ) + } catch (Exception e) { + echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" + result = false + summaryappend("Does not exist:") + } + if (result) { + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } + } + summaryappend("${artifactfilter},") + } + return result +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + + return changed +} + +def ArrayList maketaglist(buildmeta, String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + def componentothertagitem = "${othertagitem.trim()}_${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + taglist.add(componentothertagitem) + } + } + return taglist +} + +def String releasegitgetshacode(String tag) { + def existingshacode = "" + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + existingshacode = readFile(file: gitrevparseoutputfile).trim() + } + return existingshacode +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def String existingshacode = releasegitgetshacode(tag) + if (existingshacode) { // Tag exists + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The release tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } else { + summaryappend("(${tag} - Already added)") + } + } else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def boolean isalreadyreleased(buildmeta, tag) { + def boolean alreadyreleased = buildmeta.releaseinfo.released + if (alreadyreleased) { + echo "Component is already marked as released: ${alreadyreleased}" + } else { + String commitshacode = releasegitgetshacode(tag) + if (commitshacode) { + echo "Tag: ${tag} already exists in this repo therefore marking as already released" + alreadyreleased = true + } else { + alreadyreleased = false + } + } + return alreadyreleased +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { + summaryappend(" Git operations: ") + def String buildstamptag = buildmeta.releasetag() + + dir(releaserepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag) + + def String postcommitshacode + def boolean changed = false + if (alreadyreleased) { + summaryappend("Already-released,") + postcommitshacode = releasegitgetshacode(buildstamptag) + echo "Previous release sha code: ${postcommitshacode}" + } else { + releasegitadd() + + changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamptag) + } else { + releasegitreset() + summaryappend("No-commit,") + } + + postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + } + + ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } + dir(sourcerepo) { + def String srcbuildstamptag = buildmeta.globalreleasetag() + def String buildsha = buildmeta.gitinfo.gitrevision + + if (buildsha) { + def ArrayList srctaglist = maketaglist(buildmeta, srcbuildstamptag, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } else { + summaryappend(",No Src ") + } + } +} + +def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { + def buildmeta = buildinfo.meta + def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) + + checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) + + if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + } + + summaryappend("\n") +} + +def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { + def int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + versioncomponent += "${key}\n" + } + echo versioncomponent + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfoVersionUnique = buildmeta.releasewalkerversionunique(md) + echo "Version unique components:" + def String versionuniquecomponent = "" + for (key in componentReleaseInfoVersionUnique.keySet()) { + versionuniquecomponent += "${key}\n" + } + echo versionuniquecomponent + + if (DO_INTEGRITY_CHECK == "true") { + if (componentReleaseInfo.size() != componentReleaseInfoVersionUnique.size()) { + error "There are 2 or more versions of the same component being used in this hierarchy: Is this correct?" + } + } + + def toplevelid = buildmeta.componentidentifier() + // For each component + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + if (toplevelid != key) { + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) + } + } + // Do the top level last + def toplevelbuildinfo = componentReleaseInfo[toplevelid] + releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..bd6a528 --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { + if (releaserepo?.trim()) { + vc.checkRepoExists(releaserepo) + } + meta.releaseinfo.set(releaserepo) +} + +return this; diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..79c0234 --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,89 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + // Place the top level build stamp on all + OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/test.gsh b/test.gsh deleted file mode 100644 index 23fcfc0..0000000 --- a/test.gsh +++ /dev/null @@ -1,7 +0,0 @@ -// Methods in this file will end up as object methods on the object that load returns. -def lookAtThis(String whoAreYou) { - echo "Look at this, ${whoAreYou}! You loaded this from another file!" -} - -return this; - diff --git a/textutils/subst.groovy b/textutils/subst.groovy new file mode 100644 index 0000000..055a269 --- /dev/null +++ b/textutils/subst.groovy @@ -0,0 +1,8 @@ + +def substitute(String filepath, String pattern, String subst) { + fileText = readFile(file: filepath, encoding: 'UTF-8') + def converted = fileText.replaceAll(pattern, subst) + writeFile(file: filepath, text: converted, encoding: 'UTF-8') +} + +return this; \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy new file mode 100644 index 0000000..bb7c0ce --- /dev/null +++ b/versioncontrol/git.groovy @@ -0,0 +1,365 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") + +def boolean isShallowClonedRepo(String gitrootpath) { + def boolean isshallowclone = true + + dir( gitrootpath) { + if (isUnix()) { + isshallowclone = sh(returnStdout: true, script: "git rev-parse --is-shallow-repository").trim().toBoolean() + } + else { + isshallowclone = bat(returnStdout: true, script: "@git rev-parse --is-shallow-repository").trim().toBoolean() + } + echo "Repo at ${gitrootpath} is shallow cloned: ${isshallowclone}" + } + return isshallowclone +} + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git rev-parse HEAD') + } +} + +@NonCPS +def ArrayList sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return sparsecheckoutpaths +} + +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } else { + def reporootname = uniquereporootname(gitURL) + reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootsubfolder}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status" + dir(reporootsubfolder) { + if (isUnix()) { + sh """ + git status + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + bat """ + git status + if errorlevel 1 EXIT /b 1 + """ + } + } + } + } + return reporootsubfolder +} + +def checkgitlfsinstall() { + if (isUnix()) { + sh """ + git lfs install + if [ \$? -ne 0 ]; then exit 1; fi + """ + + } else { + bat """ + git lfs install + if errorlevel 1 EXIT /b 1 + """ + } +} + +//No credentials specified +//Cloning the remote Git repository +//Cloning repository git@hydra:print-server-releases/print_server_whole.git +//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10 +//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git +//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 +// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// No valid HEAD. Skipping the resetting +// > git clean -fdx # timeout=10 +// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git +// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120 +// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10 +// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_7" +// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10 + + +// > git rev-parse --is-inside-work-tree # timeout=10 +// Fetching changes from the remote Git repository +// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10 +// Cleaning workspace +// > git rev-parse --verify HEAD # timeout=10 +// Resetting working tree +// > git reset --hard # timeout=10 +// > git clean -fdx # timeout=10 +// Using shallow fetch with depth 1 +// Fetching upstream changes from git@hydra:releases/print-server-installer.git +// > git --version # timeout=10 +// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120 +// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10 +// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10 +// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master) +// Enabling Git LFS pull +// > git config core.sparsecheckout # timeout=10 +// > git read-tree -mu HEAD # timeout=10 +// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120 +// > git config --get remote.origin.url # timeout=10 +// > git lfs pull origin # timeout=120 +// Commit message: "2.10180.7389.21125_3" +// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10 + +def String lfsinstallandreporoot(String gitURL) { + checkgitlfsinstall() + return checkouttoreporoot(gitURL) +} + +def showsparseitemlist(ArrayList sparseitemlist = new ArrayList()) { + if (!sparseitemlist.isEmpty()) { + def itemstring = "Items in sparse item list:\n" + for (item in sparseitemlist) { + itemstring += item + "\n" + } + echo itemstring + } +} + +def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) { + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)], + ], + submoduleCfg: [], + ]) +} + +// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt() +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) +} + +def gitinit(String targetDir, String reporootpath) { + if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist + echo "git init ${targetDir}" + if (isUnix()) { + sh """ + git init "${targetDir}" + """ + } else { + bat """ + git init "${targetDir}" + """ + } + } + echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates" + dir("${targetDir}/.git/objects/info") { + writeFile(file: 'alternates', text: "${reporootpath}/.git/objects") + } +} + +def setlfsfetchinclude(String targetDir, ArrayList sparseitemlist) { + def String sparseitemliststring = "" + if (!sparseitemlist.isEmpty()) { + for (item in sparseitemlist) { + sparseitemliststring += "${item}," + } + echo "git config lfs.fetchinclude ${sparseitemliststring}" + dir(targetDir) { + if (isUnix()) { + sh """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } else { + bat """ + git config lfs.fetchinclude "${sparseitemliststring}" + """ + } + } + } +} + +def gitsparserestore(String targetDir) { + echo "Clean out full fetch; Force a sparse fetch" + dir(targetDir) { + if (isUnix()) { + sh """ + git rm -rf . > /dev/null + git clean -fxd > /dev/null + git reset --hard > /dev/null + """ + } else { + bat """ + git rm -rf . 2> nul + git clean -fxd 2> nul + git reset --hard 2> nul + """ + } + } + } + +def List convertSparseItemsString(String itemstopublish) { + def List sparsepaths = [] + + boolean nonempty = itemstopublish?.trim() + if (nonempty) { + def String[] items = itemstopublish.trim().split(',') + def String allpaths = "" + for(String item : items) { + sparsepaths.add(item) + allpaths += "${item}\n" + } + echo "Selected item to publish:\n${allpaths}" + } + return sparsepaths +} + +def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + echo "Performing alternative checkout" + def String reporootpath = lfsinstallandreporoot(gitURL) + showsparseitemlist(sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitinit(targetDir, reporootpath) + setlfsfetchinclude(targetDir, sparseitemlist) + } + docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist) + if (!sparseitemlist.isEmpty()) { + gitsparserestore(targetDir) + } +} + +def checkRepoExists(String gitURL) { + def int result = 0 + if (isUnix()) { + result = sh( script: """ + git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + """, + returnStatus: true + ) + } else { + result = bat( script: """ + @git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY + """, + returnStatus: true + ) + } + echo "git ls-remote return result: ${result}" + if (result > 0) { + error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building") + } else { + echo "The release repo: ${gitURL} exists" + } +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } + else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +def stripgithistory(String rootfolder) { + echo "Stripping git history out of ${rootfolder}" + dir(rootfolder) { + if (isUnix()) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + find . -iname "*@tmp" | xargs -iXX rm -rf XX + """ + } else { + echo "Not supporting git stripping in Windows at the moment" + } + } +} + +return this diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh deleted file mode 100644 index dbc09ff..0000000 --- a/versioncontrol/git.gsh +++ /dev/null @@ -1,63 +0,0 @@ -clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") - -String getGitInfo(String workingSetRootPath) { - dir(workingSetRootPath) { - return clh.getCommandOutput('git log -1 --format=oneline') - } -} - -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" - } else { - reporootpath = "" - echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" - } - checkout([ - $class: 'GitSCM', - userRemoteConfigs: [ - [url: gitURL] - ], - branches: [ - [name: revision] - ], - doGenerateSubmoduleConfigurations: false, - extensions: [ - [$class: 'GitLFSPull'], - [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] - ], - submoduleCfg: [], - ]) -} - -String getSHAFromVersion(String versionFolder) { - dir (versionFolder) { - srcRevision = readFile("gitrevision.txt").trim() - } - return srcRevision -} - -boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { - def keys = versions.keySet() - if (keys.size() <= 1) { - return false - } else { - version = "" - for (key in keys) { - nextversion = versions.get(key) - if (version == "") { - version = nextversion - } - if (nextversion != version) { - return false - } - } - } - return true -} - -return this diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..666aee3 --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,23 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + def filetoconvert = convertdetail[0] + def pattern = convertdetail[1] + def replacement = convertdetail[2] + //echo "Converting any instances of pattern: ${pattern} with replacement: ${replacement} in file: ${filetoconvert}" + def stampscript = "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" + //echo "Using python: ${PYTHON} and script: ${stampscript}" + if (isUnix()) { + sh """ + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + bat """ + "${PYTHON}" -u "${stampscript}" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + } +} + +return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..2e4f60d --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,162 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} + +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" + return newrevision +} + +def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl + + def boolean isShallowCloned = gh.isShallowClonedRepo(gitrootpath) + if (isShallowCloned) { + error("Cannot generate a version number from a shallow cloned repo at ${gitrootpath}") + } + + dir( gitrootpath) { + def int changecount = 0 + if (isUnix()) { + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() + } + else { + changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() + } + echo "Change count: ${changecount}" + def int mvnp1 = maxversionnumber() + 1 + basebuildnumber = (basebuildnumber < 0) ? mvnp1 - (basebuildnumber.abs() % mvnp1) : basebuildnumber + adjustedchangecount = incrementrevision(changecount, basebuildnumber) + echo "Adjusted change count: ${adjustedchangecount}" + if (isUnix()) { + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + } + else { + gitrevision = bat(returnStdout: true, script: "@git rev-parse HEAD").trim() + } + echo "Revision: ${gitrevision}" + if (isUnix()) { + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + } + else { + remoteinfo = bat(returnStdout: true, script: "@git remote show origin").trim() + } + echo "Remote info: ${remoteinfo}" + if (isUnix()) { + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + else { + remoteurl = bat(returnStdout: true, script: "@git config --get remote.origin.url").trim() + } + echo "Remote URL: ${remoteurl}" + } + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] +} + +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + + def defaultversionprefix = "1.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } + + def mastertypeversionprefix = "1.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + + def splits2 = sourcebranch.split('\\.') + if (splits2.size() < 1) { + def thirdpartyversionprefix = "1.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + + for (split in splits2) { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + } + + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + int integer = integerstring.toInteger() + integer = integer % (maxversionnumber() + 1) + integers.add(integer) + } + + def versionprefix = "" + if (integers.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { + break + } + } + if (integernumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." + } + } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} + +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) + + def versionprefix = getVersionPrefix(sourcerevision) + + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" + + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) + + return buildstamp +} + +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { + def int bbn = basebuildnumber.trim().toInteger() + return generate(meta, sourcerevision, sourcepath, bbn) +} + +return this; \ No newline at end of file diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py deleted file mode 100644 index a9d8738..0000000 --- a/versionnumbering/versioninfo.py +++ /dev/null @@ -1,219 +0,0 @@ -# encoding: utf-8 -''' -versionnumbering -- shortdesc - -versionnumbering generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -slaveMachine = os.getenv("NODE_NAME", default="-") - -def resetFolder(versionfolder): - print("Removing folders and contents of: {0}".format(versionfolder)) - try: - shutil.rmtree(versionfolder) - except FileNotFoundError: - pass - - print("Creating folder: {0}".format(versionfolder)) - os.makedirs(versionfolder) - -def getGitRevision(gitrootpath, basenumber): - cwd = os.getcwd() - - path = os.path.expanduser(gitrootpath) - print("Changing current working directory to: {0}".format(path)) - os.chdir(path) - - print("Determining revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - changecount = completed.stdout.decode("utf-8").strip() - print("Change count: {0}".format(changecount)) - adjustedchangecount = int(changecount) + int(basenumber) - print("Adjusted change count: {0}".format(adjustedchangecount)) - - print("Determining current revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - revision = completed.stdout.decode("utf-8").strip() - print("Revision: {0}".format(revision)) - - print("Determining remote info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteinfo = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteinfo)) - - os.chdir(cwd) - print("Restored current working directory to: {0}".format(cwd)) - - return (adjustedchangecount, revision, remoteinfo) - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().endswith("master"): # Special case - return mastertypeversionprefix - - splits = sourcebranch.split('.') - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") - parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - sourcebranch = args.sourcebranch - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber - - print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) - - resetFolder(versionfolder) - - (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 3e59c6f..4c380a6 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -26,15 +26,18 @@ def main(argv): print("Could not find file: {0}\n".format(filetoconvert)) return 1 + count = 0 with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: fileread = c.read() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) - #print ("DEBUG: %s "%converted) - print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) f.write(converted) + if count == 0: + return 1 return 0 if __name__ == "__main__":