diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..302b4f8 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -28,25 +28,54 @@ def doKillLeftovers() { } } +def doCheck(checkpathlist, build_variant = "") { + def result = 0 + for (checkpath in checkpathlist) { + echo "Checking file at: ${checkpath}" + script { + echo "Build variant is ${build_variant}" + if (build_variant) { + def srcFile = "${WORKSPACE}/${checkpath}" + def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + // Print out dependencies of the built file + echo "Showing dependencies of ${srcFileExpanded}" + result = sh( script: """ + readelf -d "${srcFileExpanded}" | grep NEEDED + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + if (result != 0) { + return result + } + } + } + } + return result +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 + def destFolder = "" + def srcFile = "" + def srcFileExpanded = "" for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + destFolder = "${exportfolder}/${build_variant}" + srcFile = "${WORKSPACE}/${exportpath}" + srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" - env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + destFolder = "${exportfolder}/${project}" + srcFileExpanded = "${WORKSPACE}/${exportpath}" } } - dir (env.destFolder) { + dir (destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { result = sh( script: """ @@ -60,10 +89,10 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } } else { - result = bat( script: ''' - copy \"%srcFileExpanded%\" \"%destFolder%\" + result = bat( script: """ + copy "${srcFileExpanded}" "${destFolder}" if errorlevel 1 EXIT /b 1 - ''', + """, returnStatus: true ) if (result != 0) { @@ -162,6 +191,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } + result = doCheck(exportpathlist, build_variant) + if (result != 0) { + error "Failed to check files...." + } result = doExport(exportfolder, exportpathlist, "", build_variant) if (result != 0) { error "Failed to copy...." diff --git a/meta/folder.groovy b/meta/folder.groovy index f9d4bf3..d751f5e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ import java.nio.file.Paths import java.nio.file.Path +import groovy.json.JsonSlurperClassic abstract class GISJsonSerialised implements Serializable { def script @@ -552,7 +553,14 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } - + + def deserialiseJsonSlurper(String filepath) { + File jsonfile = new File(filepath) + def jsonfiletext = jsonfile.getText() + def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext) + deserialiseJson(rawmeta) + } + String relativize(String mypath) { String pathRelative script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") @@ -726,4 +734,10 @@ MetaInfo metafromfolder(String folder = "") { return buildmeta } +MetaInfo metafromfolderSlurper(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + return this; diff --git a/publish/.classpath b/publish/.classpath new file mode 100644 index 0000000..287c1ab --- /dev/null +++ b/publish/.classpath @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/publish/.gitignore b/publish/.gitignore new file mode 100644 index 0000000..abed335 --- /dev/null +++ b/publish/.gitignore @@ -0,0 +1,7 @@ +/do$_publish_closure4.class +/do$_resetfolders_closure1.class +/do$_resetfolders_closure2.class +/do$_resetfolders_closure3.class +/do$_stripgithistory_closure5.class +/do.class +/publish_item_select.class diff --git a/publish/.project b/publish/.project new file mode 100644 index 0000000..5c60328 --- /dev/null +++ b/publish/.project @@ -0,0 +1,18 @@ + + + publish_item_select + + + + + + org.eclipse.jdt.core.javabuilder + + + + + + org.eclipse.jdt.groovy.core.groovyNature + org.eclipse.jdt.core.javanature + + diff --git a/publish/.settings/org.eclipse.jdt.groovy.core.prefs b/publish/.settings/org.eclipse.jdt.groovy.core.prefs new file mode 100644 index 0000000..1e68ed6 --- /dev/null +++ b/publish/.settings/org.eclipse.jdt.groovy.core.prefs @@ -0,0 +1,3 @@ +eclipse.preferences.version=1 +groovy.compiler.level=25 +groovy.script.filters=**/*.dsld,y,**/*.gradle,n diff --git a/publish/do.groovy b/publish/do.groovy index 7844289..1a54b23 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,3 +1,5 @@ +import groovy.io.FileType + markunstable = false unstablereason = "" @@ -5,6 +7,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" +filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" tempfolder = "${WORKSPACE}/temp" def resetfolders() { @@ -14,11 +17,26 @@ def resetfolders() { dir(releaserepofolder) { deleteDir() } + dir(filteredreleaserepofolder) { + deleteDir() + } dir(tempfolder) { deleteDir() } } +def productuniquefolder(releasemeta) { + return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" +} + +def uniquereleasefolder(String softwarerelease, releasemeta) { + return "Software/${PRODUCT_NAME} ${softwarerelease}/${productuniquefolder(releasemeta)}" +} + +def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) { + return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}" +} + def preconditions() { markunstable = false unstablereason = "" @@ -37,8 +55,13 @@ def summaryappend(String summary) { summarystring += summary } +def guessreporootsuffix(componentURL) { + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + return guess +} + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -46,20 +69,28 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String componentURL = releasemeta.releaseinfo.releaserepoURL def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}" def publishableitems = releasemeta.publishableinfo - def ArrayList publishablePaths = new ArrayList() + + def ArrayList publishablepaths = new ArrayList() if (publishableitems.size() > 0) { for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}" + echo "publishableitem: ${productuniquepublishableitem}" + if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) { + publishablepaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone + } + if (publishablepaths.size() > 0) { + def reporootsuffix = guessreporootsuffix(componentURL) + gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone } else { summaryappend("None ") } } -def publish(String customer) { +def publish(String customer, String finalreleaserepofolder = releaserepofolder) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' @@ -67,24 +98,37 @@ def publish(String customer) { def String FTP_USER='ftp@globalinkjetsystems.com' def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' - dir(releaserepofolder) { + dir(finalreleaserepofolder) { summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash - ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ summaryappend(", Transferred ") } } -def recallcomponent(releaseinfo, String softwarerelease) { +def copydirectory(File dirFrom, File dirTo) { + if (!dirTo.exists()) { + dirTo.mkdirs(); + } + dirFrom.eachFile(FileType.FILES) { File source -> + File target = new File(dirTo, source.getName()) + target.bytes = source.bytes + } + dirFrom.eachFile(FileType.DIRECTORIES) { File source -> + File target = new File(dirTo, source.getName()) + copyLargeDir(source, target) + } +} + +def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta - def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" - def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" - def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) + def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) summaryappend("\n") } @@ -102,11 +146,28 @@ def stripgithistory() { } } -def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { +def convertitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + def String[] pathitems = item.split('#') + def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}" + paths.add(path) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + +// Publish selected items from a release to the customer folder on the ftp site +// itemstopublishstring is a comma-seperated set of strings in this format ## +def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() preconditions() - + // Walk the meta information to a max depth, looking for artifacts to publish def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md) @@ -117,9 +178,10 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin } echo versionuniquecomponent + def itempathstopublish = convertitemstopublish(itemstopublishstring) for (key in componentReleaseInfoVersionUnique.keySet()) { def releaseinfo = componentReleaseInfoVersionUnique[key] - recallcomponent(releaseinfo, softwarerelease) + recallcomponent(releaseinfo, softwarerelease, itempathstopublish) } stripgithistory() publish(customer) @@ -127,4 +189,8 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin return [summarystring, markunstable, unstablereason] } +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + return publishhierarchy(releasemeta, "", customer, softwarerelease, maxdepth) +} + return this; diff --git a/publish/publish.groovy b/publish/publish.groovy index a703c9e..b478a5d 100644 --- a/publish/publish.groovy +++ b/publish/publish.groovy @@ -44,7 +44,7 @@ pipeline { steps { script { def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") - (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) if (markunstable) { env.unstablereason = unstablereason currentBuild.result = 'UNSTABLE' diff --git a/publish/publish_item_select.groovy b/publish/publish_item_select.groovy new file mode 100755 index 0000000..99e424f --- /dev/null +++ b/publish/publish_item_select.groovy @@ -0,0 +1,135 @@ +#!/usr/local/bin/groovy + +jobnameext = jobname ?: "jobunique" +metarepoext = metarepo ?: "git@hydra:print-server-releases/print_server_whole.git" +outputselectext = outputselect ?: "description" +metalevelsstringext = metalevels ?: "2" + +//jobnameext = "jobunique" +//metarepoext = "git@hydra:print-server-releases/print_server_whole.git" +//outputselectext = "description" +//metalevelsstringext = "2" + +metalevelsext = metalevelsstringext.toInteger() +dolog = false + +commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git" + +def log(message) { + if (dolog) { + println message + } +} + +def createfolder(String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + log("Creating directory: ${folder}") + def s = ['mkdir', '-p', folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def gitCloneOrPull(String repo, String folder) { + def sout = new StringBuilder() + def serr = new StringBuilder() + + if (!(new File(folder).exists())) { + log("Cloning: ${repo} to: ${folder}") + createfolder(folder) + def s = ['git', 'clone', repo, folder].execute() + s.consumeProcessOutput(sout, serr) + s.waitFor() + } else { + log("Pull in: ${folder}") + def s = ['git', 'pull'].execute(null, new File(folder)) + s.consumeProcessOutput(sout, serr) + s.waitFor() + } + + log("Out: ${sout} Err: ${serr}") + return [sout, serr] +} + +def loadmetaclass(String commonscripts, String metafolder) { + def metafolderscriptfile = "${commonscripts}/meta/folder.groovy" + + GroovyShell shell = new GroovyShell() + def metafolderscript = shell.parse(new File(metafolderscriptfile)) + def meta = metafolderscript.metafromfolderSlurper(metafolder) + return meta +} + +def makeworking() { + def r = ['whoami'].execute() + r.waitFor() + def user = r.text.trim() + + def rootfolder = "/home/${user}/tmp" + createfolder(rootfolder) + + def commonscripts = "${rootfolder}/commonscripts" + gitCloneOrPull(commonscriptsrepo, commonscripts) + + // Clone or pull the relevant meta repo + def metareporoot = "${rootfolder}/${jobnameext}" + gitCloneOrPull(metarepoext, metareporoot) + def metafolder = "${metareporoot}/meta" + + // Load the meta folder class that interprets the meta repo + def meta = loadmetaclass(commonscripts, metareporoot) + return meta +} + +def String outputDescription(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}") + } +} + +def String outputSelection(menuitems, releaseinfo) { + meta = releaseinfo.meta + + for (publishableitem in meta.publishableinfo) { + menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}") + } +} + +def List getItems(meta) { + def List menuitems = new ArrayList() + def metaitems = meta.releasewalkerunique(metalevelsext) + log("metaitems is ${metaitems.getClass()}") + + try { + for (key in metaitems.keySet()) { + def releaseinfo = metaitems[key] + if (outputselectext == "description") { + outputDescription(menuitems, releaseinfo) + } + if (outputselectext == "selection") { + outputSelection(menuitems, releaseinfo) + } + } + return menuitems + } catch (Exception e) { + menuitems.add("Error ${e}") + return menuitems + } +} + +def meta = makeworking() +def items = getItems(meta) + +count = 0 +for (item in items) { + count += 1 + log ("${count}: ${item}") +} + +return items \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f7d10bb..9a313f2 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,6 +19,16 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } +def guessreporoot(componentURL) { + if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { + return '' + } + def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') + def reporoot = "${REPO_ROOT}/${guess}" + echo "Guessing reporoot is: ${reporoot}" + return reporoot +} + def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRootSuffix != "") { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 4761174..f4f1bdc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -56,7 +56,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // Handle: // origin/master // origin/2.0.x -def getVersionPrefix(String sourcebranch) { +def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) { echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -121,6 +121,9 @@ def getVersionPrefix(String sourcebranch) { versionprefix += "0." } } + if (droptrailingdot == true) { + versionprefix = versionprefix.substring(0, versionprefix.length() - 1); + } echo "Determined version prefix to be: ${versionprefix}" return versionprefix }