diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0ca5da7..043c4de 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -9,11 +9,12 @@ def combineSubjobArtifacts(meta, subJobs) { if (subJobObject != null) { def subJobName = subJobObject.getProjectName() def subJobNumber = subJobObject.getNumber() - destFolder = subJob[1] + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) artifactFolders.add("${destFolder}/**") } } diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..63ca2de 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -32,7 +32,7 @@ def getmetafolderscript() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, String repoRootSuffix = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -48,7 +48,29 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def rr = load("${commonscripts()}/release/registerrepo.groovy") - rr.registerrepo(meta, releaserepo) + rr.registerrepo(meta, releaserepo, repoRootSuffix) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +// Start up boilderplate +def startnonbuild(script, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting non-build job...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") + + conditionalclean() + + def meta = getmetafolderscript().metareset(script) + meta.productinfo.set("Non-build-product", "Non-build-subcomponent", "Non-build-jira-key") + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..34eed98 --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,86 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..7bfe2d1 --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,77 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + + // Walk the meta information to a max depth + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..302b4f8 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -28,25 +28,54 @@ def doKillLeftovers() { } } +def doCheck(checkpathlist, build_variant = "") { + def result = 0 + for (checkpath in checkpathlist) { + echo "Checking file at: ${checkpath}" + script { + echo "Build variant is ${build_variant}" + if (build_variant) { + def srcFile = "${WORKSPACE}/${checkpath}" + def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + // Print out dependencies of the built file + echo "Showing dependencies of ${srcFileExpanded}" + result = sh( script: """ + readelf -d "${srcFileExpanded}" | grep NEEDED + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + if (result != 0) { + return result + } + } + } + } + return result +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 + def destFolder = "" + def srcFile = "" + def srcFileExpanded = "" for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + destFolder = "${exportfolder}/${build_variant}" + srcFile = "${WORKSPACE}/${exportpath}" + srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" - env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + destFolder = "${exportfolder}/${project}" + srcFileExpanded = "${WORKSPACE}/${exportpath}" } } - dir (env.destFolder) { + dir (destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { result = sh( script: """ @@ -60,10 +89,10 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } } else { - result = bat( script: ''' - copy \"%srcFileExpanded%\" \"%destFolder%\" + result = bat( script: """ + copy "${srcFileExpanded}" "${destFolder}" if errorlevel 1 EXIT /b 1 - ''', + """, returnStatus: true ) if (result != 0) { @@ -162,6 +191,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } + result = doCheck(exportpathlist, build_variant) + if (result != 0) { + error "Failed to check files...." + } result = doExport(exportfolder, exportpathlist, "", build_variant) if (result != 0) { error "Failed to copy...." diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..6e97510 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,61 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/logic", + "${WORKSPACE}/embedded_logic", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying logic artifacts') { + steps { + script { + env.LOGICDEST = "${WORKSPACE}/logic" + if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["embedded_logic/**"]) + sh """ + rm -v -rf meta + mv -v logic/* . + """ + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} diff --git a/differences/file.groovy b/differences/file.groovy new file mode 100644 index 0000000..ab4102f --- /dev/null +++ b/differences/file.groovy @@ -0,0 +1,16 @@ +def comparefiles(importedfile, currentfile, boolean ignorewhitespace = true) { + if (isUnix()) { + def wsparameter = "-w" + if (!ignorewhitespace) { + wsparameter = "" + } + sh """ + diff -c ${wsparameter} "${importedfile}" "${currentfile}" + """ + } + else { + error "Windows not currently supporting file comparison" + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy index 19b7cf6..5ac37f2 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -1,12 +1,10 @@ def comparefiles(importedjsonfile, currentjsonfile) { - env.IMPORTED_JSON="${importedjsonfile}" - env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ which jq if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output - diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + diff -c <(jq -S . "${importedjsonfile}") <(jq -S . "${currentjsonfile}") """ } else { diff --git a/import/importer.groovy b/import/importer.groovy index 1f89426..34707bc 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,6 +1,7 @@ +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -10,11 +11,13 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -27,8 +30,62 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } +def rewritemetafileasreleased(meta, String filepath) { + // Modify the meta info to say that is has already been released + meta.markreleased() + meta.saveJsonSerialised(filepath) +} + +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + gh.gitCheckout(rooturl, rootrevision, destpath) + + def importedmeta = meta.importrelated(destpath) + + relatedimportinfo = importedmeta.getassociated() + echo "Cloning associated repos" + for(relatedmetainfo in relatedimportinfo) { + def relatedmeta = relatedmetainfo.meta + def relatedoffsetpath = relatedmetainfo.offsetpath + def cloneurl = relatedmeta.releaseinfo.releaserepoURL + def clonerevision = "refs/tags/${relatedmeta.releasetag()}" + def clonepath = "${destpath}/${relatedoffsetpath}" + echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" + gh.gitCheckout(cloneurl, clonerevision, clonepath) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" + echo "Marking related meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(relatedmeta, metafilepath) + } + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + switch (source) { + case "Jenkins job artifact": + return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) + case "Release repository": + return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + return this; diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 093ddef..819a580 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -4,10 +4,10 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { - def artifactKey = installerDetails.get(installer)[0] - (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + def artifactKey = installerDetails.get(installer)[1] + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) artifactFetched.put(artifactKey, importmeta) } } @@ -17,12 +17,12 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { // Export any required binaries to the /bin folder def placeBinaries(artifactDetails, installerDetails) { for (installer in toBuild) { - (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + (installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) if (exportBinaries) { - (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet) for (binaryFolder in binariesFolders) { - binariesSourceFolder = "${destinationFolder}/${binaryFolder}" - binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + def binariesSourceFolder = "${destFolder}/${binaryFolder}" + def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) } diff --git a/meta/folder.groovy b/meta/folder.groovy index 2ef8e22..346b2ee 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ import java.nio.file.Paths import java.nio.file.Path +import groovy.json.JsonSlurperClassic abstract class GISJsonSerialised implements Serializable { def script @@ -118,8 +119,8 @@ class GitInfo extends GISJsonSerialised { } class JenkinsBuildInfo extends GISJsonSerialised { - private String buildjobname - private int buildnumber + String buildjobname + int buildnumber private String buildmachine private String builddate @@ -157,7 +158,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { } class BuildStampInfo extends GISJsonSerialised { - private String buildstampprefix + String buildstampprefix int revision String buildstamp @@ -218,11 +219,13 @@ class BuildStampInfo extends GISJsonSerialised { class ReleaseInfo extends GISJsonSerialised { String releaserepoURL ArrayList releaseitems + boolean released - ReleaseInfo(script, releaserepoURL = "") { + ReleaseInfo(script, String releaserepoURL = "", boolean released = false) { super(script) this.releaserepoURL = releaserepoURL this.releaseitems = new ArrayList() + this.released = released } void set(String releaserepoURL = "") { @@ -233,6 +236,10 @@ class ReleaseInfo extends GISJsonSerialised { releaseitems.add(releaseitem) } + void markreleased() { + this.released = true + } + List releaseitemssserialised() { def items = [] for (ri in releaseitems) { @@ -247,12 +254,14 @@ class ReleaseInfo extends GISJsonSerialised { for (rawreleaseitem in rawmeta['releaseitems']) { releaseitems.add(rawreleaseitem) } + released = rawmeta['released'] ?: false } LinkedHashMap serialisationdata() { def data = [ 'releaserepoURL' : releaserepoURL, 'releaseitems' : releaseitemssserialised(), + 'released' : released ] return data } @@ -419,9 +428,11 @@ class PublishableInfo extends GISJsonSerialised { class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" + private static String DIFFFOLDERNAME = "diff" private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String DIFFARTIFACTS = "${DIFFFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -474,6 +485,17 @@ class MetaInfo extends GISJsonSerialised { associatedmeta = new ArrayList() } + List getassociated() { + return associatedmeta + } + + void markreleased() { + releaseinfo.markreleased() + for (associated in associatedmeta) { + associated.meta.releaseinfo.markreleased() + } + } + List configurationinfoserialised() { def configuration = [] for (configurationitem in configurationinfo) { @@ -484,6 +506,7 @@ class MetaInfo extends GISJsonSerialised { void addPublishable(String filepath) { def PublishableInfo pi = new PublishableInfo(script, filepath) + script.echo("Adding ${filepath} to publishable list") publishableinfo.add(pi) } @@ -549,15 +572,32 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } - - String relativize(String path) { - def Path thispath = Paths.get(path) - if (thispath.startsWith(script.WORKSPACE)) { - def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(thispath) - return wsrelativePath.toString() + + def deserialiseJsonSlurper(String filepath) { + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) + deserialiseJson(rawmeta) + } + + String relativize(String mypath) { + String pathRelative + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") + } else { + pathRelative = mypath + script.echo("Path relative is unchanged: ${pathRelative}") } - return path + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') + // Strip leading / if any + pathRelative = pathRelative.replaceAll('^/+', '') + + script.echo("Path relative is ${pathRelative}") + + return pathRelative } static String metafilepath() { @@ -565,7 +605,7 @@ class MetaInfo extends GISJsonSerialised { } static String difffilepath() { - return METAFOLDERNAME + "/" + METADIFFFILENAME + return DIFFFOLDERNAME + "/" + METADIFFFILENAME } MetaInfo importrelated(String destpath) { @@ -610,17 +650,16 @@ class MetaInfo extends GISJsonSerialised { ] return data } - static String folder() { return METAFOLDERNAME } - + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { - def String artifactlist = METAARTIFACTS + def String artifactlist = "${METAARTIFACTS},${DIFFARTIFACTS}" releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { artifactlist += "," + releaseitem @@ -663,9 +702,29 @@ class MetaInfo extends GISJsonSerialised { } String componentversionuniqueidentifier() { - return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp + return componentidentifier() + "_" + buildstampinfo.buildstamp + } + + String globalreleasetag() { + return componentversionuniqueidentifier() + "_" + jenkinsbuildinfo.buildnumber } + String releasetag() { + return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber + } + + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { + releasemetacollection.put(uniquestring, relatedmetainfo) + } + } else { + releasemetacollection.put(uniquestring, relatedmetainfo) + } + return releasemetacollection + } + Map releasewalkerunique(int maxdepth, String startpath = "") { def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) @@ -673,7 +732,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) } return releasemetauniquecollection @@ -686,7 +745,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetaversionuniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) } return releasemetaversionuniquecollection @@ -713,4 +772,10 @@ MetaInfo metafromfolder(String folder = "") { return buildmeta } +MetaInfo metafromfolderSlurper(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + return this; diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/do.groovy b/publish/do.groovy index 7844289..b63b15c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,3 +1,5 @@ +import groovy.io.FileType + markunstable = false unstablereason = "" @@ -5,6 +7,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" tempfolder = "${WORKSPACE}/temp" def resetfolders() { @@ -14,11 +17,26 @@ def resetfolders() { dir(releaserepofolder) { deleteDir() } + dir(filteredreleaserepofolder) { + deleteDir() + } dir(tempfolder) { deleteDir() } } +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${productuniquefolder(releasemeta)}" +} + +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + def preconditions() { markunstable = false unstablereason = "" @@ -37,8 +55,13 @@ def summaryappend(String summary) { summarystring += summary } +def guessreporootsuffix(componentURL) { + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + return guess +} + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -46,20 +69,28 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo - def ArrayList publishablePaths = new ArrayList() + + def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone + } + if (publishablepaths.size() > 0) { + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone } else { summaryappend("None ") } } -def publish(String customer) { +def publish(String customer, String finalreleaserepofolder = releaserepofolder) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' @@ -67,46 +98,63 @@ def publish(String customer) { def String FTP_USER='ftp@globalinkjetsystems.com' def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' - dir(releaserepofolder) { + dir(finalreleaserepofolder) { summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash - ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ summaryappend(", Transferred ") } } -def recallcomponent(releaseinfo, String softwarerelease) { +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdirs(); + } + dirFrom.eachFile(FileType.FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) + def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) summaryappend("\n") } -def stripgithistory() { - echo "Stripping git history out of ${releaserepofolder}" - dir(releaserepofolder) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rvf XX - find . -name ".gitignore" | xargs -iXX rm -rvf XX - find . -name ".gitmodules" | xargs -iXX rm -rvf XX - find . -iname ".gitattributes" | xargs -iXX rm -rvf XX - """ +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + allpaths += "${path}\n" } + echo "Selected item to publish:\n${allpaths}" + return paths } -def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() preconditions() - + // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) @@ -117,14 +165,19 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin } echo versionuniquecomponent + def itempathstopublish = convertitemstopublish(itemstopublishstring) for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] - recallcomponent(releaseinfo, softwarerelease) + recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } - stripgithistory() + gh.stripgithistory(releaserepofolder) publish(customer) return [summarystring, markunstable, unstablereason] } +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(releasemeta, "", customer, softwarerelease, maxdepth) +} + return this; diff --git a/publish/publish.groovy b/publish/publish.groovy index a703c9e..b478a5d 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -44,7 +44,7 @@ pipeline { steps { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy new file mode 100755 index 0000000..99e424f --- /dev/null +++ b/publish/publish_item_select.groovy @@ -0,0 +1,135 @@ +#!/usr/local/bin/groovy + +jobnameext = jobname ?: "jobunique" +metarepoext = metarepo ?: "git@hydra:print-server-releases/print_server_whole.git" +outputselectext = outputselect ?: "description" +metalevelsstringext = metalevels ?: "2" + +//jobnameext = "jobunique" +//metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +//outputselectext = "description" +//metalevelsstringext = "2" + +metalevelsext = metalevelsstringext.toInteger() +dolog = false + +commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" + +def log(message) { + if (dolog) { + println message + } +} + +def createfolder(String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + log("Creating directory: ${folder}") + def s = ['mkdir', '-p', folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def gitCloneOrPull(String repo, String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + if (!(new File(folder).exists())) { + log("Cloning: ${repo} to: ${folder}") + createfolder(folder) + def s = ['git', 'clone', repo, folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } else { + log("Pull in: ${folder}") + def s = ['git', 'pull'].execute(null, new File(folder)) + s.consumeProcessOutput(sout, serr) + s.waitFor() + } + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def loadmetaclass(String commonscripts, String metafolder) { + def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" + + GroovyShell shell = new GroovyShell() + def metafolderscript = shell.parse(new File(metafolderscriptfile)) + def meta = metafolderscript.metafromfolderSlurper(metafolder) + return meta +} + +def makeworking() { + def r = ['whoami'].execute() + r.waitFor() + def user = r.text.trim() + + def rootfolder = "/home/${user}/tmp" + createfolder(rootfolder) + + def commonscripts = "${rootfolder}/commonscripts" + gitCloneOrPull(commonscriptsrepo, commonscripts) + + // Clone or pull the relevant meta repo + def metareporoot = "${rootfolder}/${jobnameext}" + gitCloneOrPull(metarepoext, metareporoot) + def metafolder = "${metareporoot}/meta" + + // Load the meta folder class that interprets the meta repo + def meta = loadmetaclass(commonscripts, metareporoot) + return meta +} + +def String outputDescription(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } +} + +def String outputSelection(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } +} + +def List getItems(meta) { + def List menuitems = new ArrayList() + def metaitems = meta.releasewalkerunique(metalevelsext) + log("metaitems is ${metaitems.getClass()}") + + try { + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + if (outputselectext == "description") { + outputDescription(menuitems, releaseinfo) + } + if (outputselectext == "selection") { + outputSelection(menuitems, releaseinfo) + } + } + return menuitems + } catch (Exception e) { + menuitems.add("Error ${e}") + return menuitems + } +} + +def meta = makeworking() +def items = getItems(meta) + +count = 0 +for (item in items) { + count += 1 + log ("${count}: ${item}") +} + +return items \ No newline at end of file diff --git a/release/do.groovy b/release/do.groovy index 1a785b3..fd2b06b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -57,7 +57,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + sh("git clone ${releaserepo} ${checkoutfolder}") } previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { @@ -92,7 +92,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { } } -def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { @@ -104,10 +104,10 @@ def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { echo "Copying artifacts: ${filterpath} to ${tempfolder}" copyArtifacts( filter: filterpath, - projectName: BUILD_JOB, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, selector: [ $class: 'ParameterizedBuildSelector', - parameterName: BUILD_JOB_SELECTOR + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" ], target: tempfolder ) @@ -172,22 +172,29 @@ def ArrayList maketaglist(buildmeta, String buildstamp, String otherrele return taglist } -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false +def String releasegitgetshacode(String tag) { + def existingshacode = "" // Check if tag exists already def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() + existingshacode = readFile(file: gitrevparseoutputfile).trim() + } + return existingshacode +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def String existingshacode = releasegitgetshacode(tag) + if (existingshacode) { // Tag exists if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") summaryappend("(${tag} - Could not add!)") - } - else { + } else { summaryappend("(${tag} - Already added)") } - } - else { // If tag does not exist already + } else { // If tag does not exist already // Tag sh("git tag \"${tag}\"") summaryappend("(${tag})") @@ -270,26 +277,35 @@ def sourcegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp - def String buildstamptag = "${buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" + def String buildstamptag = buildmeta.releasetag() dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - releasegitadd() + def boolean alreadyreleased = buildmeta.releaseinfo.released + echo "Already released: ${alreadyreleased}" - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamptag) - } - else { - releasegitreset() - summaryappend("No-commit,") - } + def String postcommitshacode + def boolean changed = false + if (alreadyreleased) { + summaryappend("Already-released,") + postcommitshacode = releasegitgetshacode(buildstamptag) + echo "Previous release sha code: ${postcommitshacode}" + } else { + releasegitadd() - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" + changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamptag) + } else { + releasegitreset() + summaryappend("No-commit,") + } + + postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + } ArrayList taglist = maketaglist(buildmeta, buildstamptag, otherreleasetags) def tagadded = releasegittags(postcommitshacode, taglist) @@ -299,7 +315,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamptag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildstamptag}" + def String srcbuildstamptag = buildmeta.globalreleasetag() def String buildsha = buildmeta.gitinfo.gitrevision if (buildsha) { @@ -308,14 +324,13 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source if (tagadded) { sourcegitpush(dryrunpush) } - } - else { + } else { summaryappend(",No Src ") } } } -def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { +def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) { def buildmeta = buildinfo.meta def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" @@ -327,7 +342,7 @@ def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp) gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) @@ -368,12 +383,12 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] if (toplevelid != key) { - releasecomponent(buildinfo, otherreleasetags, dryrunpush) + releasecomponent(buildmeta, buildinfo, otherreleasetags, dryrunpush) } } // Do the top level last def toplevelbuildinfo = componentReleaseInfo[toplevelid] - releasecomponent(toplevelbuildinfo, otherreleasetags, dryrunpush) + releasecomponent(buildmeta, toplevelbuildinfo, otherreleasetags, dryrunpush) return [summarystring, markunstable, unstablereason] } diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index ff4d85d..2dffdc7 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -2,9 +2,9 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data -def registerrepo(meta, String releaserepo) { +def registerrepo(meta, String releaserepo, String repoRootSuffix = "") { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix) } meta.releaseinfo.set(releaserepo) } diff --git a/release/release.groovy b/release/release.groovy index b0530be..79c0234 100644 --- a/release/release.groovy +++ b/release/release.groovy @@ -43,8 +43,7 @@ pipeline { script { def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") // Place the top level build stamp on all - def topleveltag = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}_${buildmeta.jenkinsbuildinfo.buildnumber}" - OTHER_RELEASE_TAGS = "${topleveltag},${OTHER_RELEASE_TAGS}" + OTHER_RELEASE_TAGS = "${buildmeta.globalreleasetag()},${OTHER_RELEASE_TAGS}" (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) if (markunstable) { env.unstablereason = unstablereason diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 8bd7968..253ffee 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,14 +19,64 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRoot != "") { - reporootpath = "${reporootenv.trim()}/${repoRoot}" - } else { - reporootpath = "" +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } else { + def reporootname = uniquereporootname(gitURL) + reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootsubfolder}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootsubfolder} ... checking status" + dir(reporootsubfolder) { + if (isUnix()) { + sh """ + git status + if [ \$? -ne 0 ]; then exit 1; fi + """ + } else { + bat """ + git status + if errorlevel 1 EXIT /b 1 + """ + } + } + } } + return reporootsubfolder +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -78,4 +128,17 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { return true } +def stripgithistory(String rootfolder) { + echo "Stripping git history out of ${rootfolder}" + dir(rootfolder) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + """ + } +} + return this diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 4761174..f4f1bdc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -56,7 +56,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // Handle: // origin/master // origin/2.0.x -def getVersionPrefix(String sourcebranch) { +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -121,6 +121,9 @@ def getVersionPrefix(String sourcebranch) { versionprefix += "0." } } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } echo "Determined version prefix to be: ${versionprefix}" return versionprefix } diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 3e59c6f..4c380a6 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -26,15 +26,18 @@ def main(argv): print("Could not find file: {0}\n".format(filetoconvert)) return 1 + count = 0 with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: fileread = c.read() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) - #print ("DEBUG: %s "%converted) - print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) f.write(converted) + if count == 0: + return 1 return 0 if __name__ == "__main__":