Merge commit '5cac7a574b' into production

This commit is contained in:
Dan Hawkins 2019-03-19 16:19:32 +00:00
commit 240d5f6e3f
11 changed files with 326 additions and 28 deletions

View file

@ -28,25 +28,54 @@ def doKillLeftovers() {
}
}
def doCheck(checkpathlist, build_variant = "") {
def result = 0
for (checkpath in checkpathlist) {
echo "Checking file at: ${checkpath}"
script {
echo "Build variant is ${build_variant}"
if (build_variant) {
def srcFile = "${WORKSPACE}/${checkpath}"
def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}")
// Print out dependencies of the built file
echo "Showing dependencies of ${srcFileExpanded}"
result = sh( script: """
readelf -d "${srcFileExpanded}" | grep NEEDED
if [ \$? -ne 0 ]; then exit 1; fi
""",
returnStatus: true
)
if (result != 0) {
return result
}
}
}
}
return result
}
def doExport(exportfolder, exportpathlist, project = "", build_variant = "") {
def result = 0
def destFolder = ""
def srcFile = ""
def srcFileExpanded = ""
for (exportpath in exportpathlist) {
echo "Export path at: ${exportpath}"
script {
echo "Build variant is ${build_variant}"
if (build_variant) {
echo "Build variant is ${build_variant}"
env.destFolder = "${exportfolder}/${build_variant}"
env.srcFile = "${WORKSPACE}/${exportpath}"
env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}")
destFolder = "${exportfolder}/${build_variant}"
srcFile = "${WORKSPACE}/${exportpath}"
srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}")
}
else {
echo "No build variant"
env.destFolder = "${exportfolder}/${project}"
env.srcFileExpanded = "${WORKSPACE}/${exportpath}"
destFolder = "${exportfolder}/${project}"
srcFileExpanded = "${WORKSPACE}/${exportpath}"
}
}
dir (env.destFolder) {
dir (destFolder) {
echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\""
if (isUnix()) {
result = sh( script: """
@ -60,10 +89,10 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") {
}
}
else {
result = bat( script: '''
copy \"%srcFileExpanded%\" \"%destFolder%\"
result = bat( script: """
copy "${srcFileExpanded}" "${destFolder}"
if errorlevel 1 EXIT /b 1
''',
""",
returnStatus: true
)
if (result != 0) {
@ -162,6 +191,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou
"""
}
}
result = doCheck(exportpathlist, build_variant)
if (result != 0) {
error "Failed to check files...."
}
result = doExport(exportfolder, exportpathlist, "", build_variant)
if (result != 0) {
error "Failed to copy...."

View file

@ -1,5 +1,6 @@
import java.nio.file.Paths
import java.nio.file.Path
import groovy.json.JsonSlurperClassic
abstract class GISJsonSerialised implements Serializable {
def script
@ -552,7 +553,14 @@ class MetaInfo extends GISJsonSerialised {
def rawmeta = script.readJSON(file: filepath)
deserialiseJson(rawmeta)
}
def deserialiseJsonSlurper(String filepath) {
File jsonfile = new File(filepath)
def jsonfiletext = jsonfile.getText()
def rawmeta = new JsonSlurperClassic().parseText(jsonfiletext)
deserialiseJson(rawmeta)
}
String relativize(String mypath) {
String pathRelative
script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}")
@ -726,4 +734,10 @@ MetaInfo metafromfolder(String folder = "") {
return buildmeta
}
MetaInfo metafromfolderSlurper(String folder = "") {
MetaInfo buildmeta = new MetaInfo(this)
buildmeta.deserialiseJsonSlurper("${folder}/${MetaInfo.metafilepath()}")
return buildmeta
}
return this;

9
publish/.classpath Normal file
View file

@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry excluding="publish.groovy" kind="src" path=""/>
<classpathentry exported="true" kind="con" path="GROOVY_SUPPORT"/>
<classpathentry exported="true" kind="con" path="GROOVY_DSL_SUPPORT"/>
<classpathentry kind="lib" path="/home/dan/.sdkman/candidates/groovy/2.5.6/lib/groovy-json-2.5.6.jar"/>
<classpathentry kind="output" path=""/>
</classpath>

7
publish/.gitignore vendored Normal file
View file

@ -0,0 +1,7 @@
/do$_publish_closure4.class
/do$_resetfolders_closure1.class
/do$_resetfolders_closure2.class
/do$_resetfolders_closure3.class
/do$_stripgithistory_closure5.class
/do.class
/publish_item_select.class

18
publish/.project Normal file
View file

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>publish_item_select</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.groovy.core.groovyNature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>

View file

@ -0,0 +1,3 @@
eclipse.preferences.version=1
groovy.compiler.level=25
groovy.script.filters=**/*.dsld,y,**/*.gradle,n

View file

@ -1,3 +1,5 @@
import groovy.io.FileType
markunstable = false
unstablereason = ""
@ -5,6 +7,7 @@ summaryfolder = "${WORKSPACE}/summary"
summaryfile = "${summaryfolder}/summary.txt"
summarystring = ""
releaserepofolder = "${WORKSPACE}/releaserepos"
filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos"
tempfolder = "${WORKSPACE}/temp"
def resetfolders() {
@ -14,11 +17,26 @@ def resetfolders() {
dir(releaserepofolder) {
deleteDir()
}
dir(filteredreleaserepofolder) {
deleteDir()
}
dir(tempfolder) {
deleteDir()
}
}
def productuniquefolder(releasemeta) {
return "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}"
}
def uniquereleasefolder(String softwarerelease, releasemeta) {
return "Software/${PRODUCT_NAME} ${softwarerelease}/${productuniquefolder(releasemeta)}"
}
def uniqueabsoluterreleasefolder(String releaserepofolder, String softwarerelease, releasemeta) {
return "${releaserepofolder}/${uniquereleasefolder(softwarerelease, releasemeta)}"
}
def preconditions() {
markunstable = false
unstablereason = ""
@ -37,8 +55,13 @@ def summaryappend(String summary) {
summarystring += summary
}
def guessreporootsuffix(componentURL) {
def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1')
return guess
}
gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy")
def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) {
def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) {
// Copy relevant artifacts to temp folder
summaryappend("Artifacts: ")
@ -46,20 +69,28 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) {
def String componentURL = releasemeta.releaseinfo.releaserepoURL
def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}_${releasemeta.jenkinsbuildinfo.buildnumber}"
def publishableitems = releasemeta.publishableinfo
def ArrayList<String> publishablePaths = new ArrayList<String>()
def ArrayList<String> publishablepaths = new ArrayList<String>()
if (publishableitems.size() > 0) {
for (publishableitem in publishableitems) {
publishablePaths.add(publishableitem.filepath)
summaryappend("${publishableitem.filepath},")
def String productuniquepublishableitem = "${productuniquefolder(releasemeta)}/${publishableitem.filepath}"
echo "publishableitem: ${productuniquepublishableitem}"
if (itempathstopublish.size() == 0 || productuniquepublishableitem in itempathstopublish) {
publishablepaths.add(publishableitem.filepath)
summaryappend("${publishableitem.filepath},")
}
}
gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone
}
if (publishablepaths.size() > 0) {
def reporootsuffix = guessreporootsuffix(componentURL)
gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone
}
else {
summaryappend("None ")
}
}
def publish(String customer) {
def publish(String customer, String finalreleaserepofolder = releaserepofolder) {
summaryappend(" FTP operations: ")
def String FTP_CLIENT_LOCATION='ftp'
@ -67,24 +98,37 @@ def publish(String customer) {
def String FTP_USER='ftp@globalinkjetsystems.com'
def String FTP_CUSTOMER_ROOT="/Customers/${customer}"
def String FTP_PASSWORD='ramshorn42'
dir(releaserepofolder) {
dir(finalreleaserepofolder) {
summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}")
sh """#!/bin/bash
ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" .
ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" .
"""
summaryappend(", Transferred ")
}
}
def recallcomponent(releaseinfo, String softwarerelease) {
def copydirectory(File dirFrom, File dirTo) {
if (!dirTo.exists()) {
dirTo.mkdirs();
}
dirFrom.eachFile(FileType.FILES) { File source ->
File target = new File(dirTo, source.getName())
target.bytes = source.bytes
}
dirFrom.eachFile(FileType.DIRECTORIES) { File source ->
File target = new File(dirTo, source.getName())
copyLargeDir(source, target)
}
}
def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) {
def releasemeta = releaseinfo.meta
def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}"
def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}"
def String uniquefoldertemp = "${tempfolder}/${uniquefolder}"
def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta)
def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta)
summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ")
getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp)
getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish)
summaryappend("\n")
}
@ -102,11 +146,28 @@ def stripgithistory() {
}
}
def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) {
def convertitemstopublish(String itemstopublishstring) {
def List paths = []
def String[] items = itemstopublishstring.split(',')
def String allpaths = ""
for(String item : items) {
def String[] pathitems = item.split('#')
def String path = "${pathitems[0]}_${pathitems[1]}/${pathitems[2]}"
paths.add(path)
allpaths += "${path}\n"
}
echo "Selected item to publish:\n${allpaths}"
return paths
}
// Publish selected items from a release to the customer folder on the ftp site
// itemstopublishstring is a comma-seperated set of strings in this format <ProductName>#<ComponentName>#<PathToPublishableItem>
def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) {
int md = maxdepth.toInteger()
resetfolders()
preconditions()
// Walk the meta information to a max depth, looking for artifacts to publish
def componentReleaseInfoVersionUnique = releasemeta.releasewalkerversionunique(md)
@ -117,9 +178,10 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin
}
echo versionuniquecomponent
def itempathstopublish = convertitemstopublish(itemstopublishstring)
for (key in componentReleaseInfoVersionUnique.keySet()) {
def releaseinfo = componentReleaseInfoVersionUnique[key]
recallcomponent(releaseinfo, softwarerelease)
recallcomponent(releaseinfo, softwarerelease, itempathstopublish)
}
stripgithistory()
publish(customer)
@ -127,4 +189,8 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin
return [summarystring, markunstable, unstablereason]
}
def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) {
return publishhierarchy(releasemeta, "", customer, softwarerelease, maxdepth)
}
return this;

View file

@ -44,7 +44,7 @@ pipeline {
steps {
script {
def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy")
(summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH)
(summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH)
if (markunstable) {
env.unstablereason = unstablereason
currentBuild.result = 'UNSTABLE'

View file

@ -0,0 +1,135 @@
#!/usr/local/bin/groovy
jobnameext = jobname ?: "jobunique"
metarepoext = metarepo ?: "git@hydra:print-server-releases/print_server_whole.git"
outputselectext = outputselect ?: "description"
metalevelsstringext = metalevels ?: "2"
//jobnameext = "jobunique"
//metarepoext = "git@hydra:print-server-releases/print_server_whole.git"
//outputselectext = "description"
//metalevelsstringext = "2"
metalevelsext = metalevelsstringext.toInteger()
dolog = false
commonscriptsrepo="git@hydra:jenkinsautobuild/Common_scripts.git"
def log(message) {
if (dolog) {
println message
}
}
def createfolder(String folder) {
def sout = new StringBuilder()
def serr = new StringBuilder()
log("Creating directory: ${folder}")
def s = ['mkdir', '-p', folder].execute()
s.consumeProcessOutput(sout, serr)
s.waitFor()
log("Out: ${sout} Err: ${serr}")
return [sout, serr]
}
def gitCloneOrPull(String repo, String folder) {
def sout = new StringBuilder()
def serr = new StringBuilder()
if (!(new File(folder).exists())) {
log("Cloning: ${repo} to: ${folder}")
createfolder(folder)
def s = ['git', 'clone', repo, folder].execute()
s.consumeProcessOutput(sout, serr)
s.waitFor()
} else {
log("Pull in: ${folder}")
def s = ['git', 'pull'].execute(null, new File(folder))
s.consumeProcessOutput(sout, serr)
s.waitFor()
}
log("Out: ${sout} Err: ${serr}")
return [sout, serr]
}
def loadmetaclass(String commonscripts, String metafolder) {
def metafolderscriptfile = "${commonscripts}/meta/folder.groovy"
GroovyShell shell = new GroovyShell()
def metafolderscript = shell.parse(new File(metafolderscriptfile))
def meta = metafolderscript.metafromfolderSlurper(metafolder)
return meta
}
def makeworking() {
def r = ['whoami'].execute()
r.waitFor()
def user = r.text.trim()
def rootfolder = "/home/${user}/tmp"
createfolder(rootfolder)
def commonscripts = "${rootfolder}/commonscripts"
gitCloneOrPull(commonscriptsrepo, commonscripts)
// Clone or pull the relevant meta repo
def metareporoot = "${rootfolder}/${jobnameext}"
gitCloneOrPull(metarepoext, metareporoot)
def metafolder = "${metareporoot}/meta"
// Load the meta folder class that interprets the meta repo
def meta = loadmetaclass(commonscripts, metareporoot)
return meta
}
def String outputDescription(menuitems, releaseinfo) {
meta = releaseinfo.meta
for (publishableitem in meta.publishableinfo) {
menuitems.add("${meta.productinfo.productname} ${meta.productinfo.subcomponentname} ${publishableitem.filepath}")
}
}
def String outputSelection(menuitems, releaseinfo) {
meta = releaseinfo.meta
for (publishableitem in meta.publishableinfo) {
menuitems.add("${meta.productinfo.productname}#${meta.productinfo.subcomponentname}#${publishableitem.filepath}")
}
}
def List<String> getItems(meta) {
def List<String> menuitems = new ArrayList<String>()
def metaitems = meta.releasewalkerunique(metalevelsext)
log("metaitems is ${metaitems.getClass()}")
try {
for (key in metaitems.keySet()) {
def releaseinfo = metaitems[key]
if (outputselectext == "description") {
outputDescription(menuitems, releaseinfo)
}
if (outputselectext == "selection") {
outputSelection(menuitems, releaseinfo)
}
}
return menuitems
} catch (Exception e) {
menuitems.add("Error ${e}")
return menuitems
}
}
def meta = makeworking()
def items = getItems(meta)
count = 0
for (item in items) {
count += 1
log ("${count}: ${item}")
}
return items

View file

@ -19,6 +19,16 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList<String> paths) {
return new SparseCheckoutPaths(sparsecheckoutpaths)
}
def guessreporoot(componentURL) {
if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) {
return ''
}
def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1')
def reporoot = "${REPO_ROOT}/${guess}"
echo "Guessing reporoot is: ${reporoot}"
return reporoot
}
def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList<String> sparseitemlist = new ArrayList<String>()) {
reporootenv = env.REPO_ROOT
if (reporootenv != null && repoRootSuffix != "") {

View file

@ -56,7 +56,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) {
// Handle:
// origin/master
// origin/2.0.x
def getVersionPrefix(String sourcebranch) {
def getVersionPrefix(String sourcebranch, boolean droptrailingdot = false) {
echo "Determining version number prefix from source branch name: ${sourcebranch}"
sourcebranch = sourcebranch.toLowerCase()
@ -121,6 +121,9 @@ def getVersionPrefix(String sourcebranch) {
versionprefix += "0."
}
}
if (droptrailingdot == true) {
versionprefix = versionprefix.substring(0, versionprefix.length() - 1);
}
echo "Determined version prefix to be: ${versionprefix}"
return versionprefix
}