Merge branch 'master' of hydra:jenkinsautobuild/Atlas_Autobuild_Scripts into testpipelinedev

# Conflicts:
#	test/temp_ui_meta_test.groovy
This commit is contained in:
george Linsdell 2021-05-10 13:17:58 +01:00
commit 9b46b385b7
13 changed files with 346 additions and 31 deletions

View file

@ -34,8 +34,17 @@ def getmetafolderscript() {
return metafolderscript
}
def alterreleaserepo(String releaserepo, String releaserepovariantsuffix) {
def String altered = releaserepo.trim()
if (releaserepovariantsuffix?.trim()) {
altered = altered.replace("-RELEASE_REPO_VARIANT_SUFFIX-", releaserepovariantsuffix)
}
return altered
}
// Start up boilderplate
def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) {
def startbuild(script, String productname = PRODUCT_NAME, String subcomponentname = SUBCOMPONENT_NAME, String releaserepo = RELEASE_REPO, String productvariant = '', String autobuildpath = autobuildroot, String commonscriptspath = commonscriptsroot) {
echo "Starting...."
autobuildroot = autobuildpath
@ -46,14 +55,16 @@ def startbuild(script, String productname = PRODUCT_NAME, String subcomponentnam
def meta = getmetafolderscript().metareset(script)
echo "Product name: ${productname} Subcomponent name: ${subcomponentname}"
meta.productinfo.set(productname, subcomponentname)
meta.productinfo.set(productname, subcomponentname, productvariant)
def now = new Date()
now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC'))
meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString())
def rr = load("${commonscripts()}/release/registerrepo.groovy")
echo "Release repo: ${releaserepo}"
rr.registerrepo(meta, releaserepo)
def String alteredreleaserepo = alterreleaserepo(releaserepo, productvariant)
echo "Release repo: ${alteredreleaserepo}"
rr.registerrepo(meta, alteredreleaserepo)
def ch = load("${commonscripts()}/configuration/configuration.groovy")
ch.registerPipelineScriptConfiguration(meta)

View file

@ -0,0 +1,172 @@
def doDisplayLeftovers() {
}
def doKillLeftovers() {
}
def doCheck(checkpathlist, build_variant = "") {
def result = 0
for (checkpath in checkpathlist) {
echo "Checking file at: ${checkpath}"
script {
echo "Build variant is ${build_variant}"
if (build_variant) {
def srcFile = "${WORKSPACE}\${checkpath}"
def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}")
// Print out dependencies of the built file
echo "Showing dependencies of ${srcFileExpanded}"
if (srcFileExpanded.endsWith(".elf")) {
result = bat ( script: """
${CYGWINDIR}/bin/readelf -hS ${srcFileExpanded}
if errorlevel 1 EXIT /b 1
""",
returnStatus: true
)
}
if (result != 0) {
return result
}
}
}
}
return result
}
def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") {
def result = 0
def destFolder = ""
def srcFile = ""
def srcFileExpanded = ""
for (exportpath in exportpathlist) {
echo "Export path at: ${exportpath}"
script {
if (build_variant) {
echo "Build variant is ${build_variant}"
srcFile = "${WORKSPACE}\\${exportpath}"
srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}")
}
else {
echo "No build variant"
srcFileExpanded = "${WORKSPACE}\\${exportpath}"
}
destFolder = "${exportfolder}"
if (partdest) {
destFolder += "\\" + partdest
}
}
def destFolderExpanded = destFolder.replace("-BUILDVARIANT-", "${build_variant}")
dir (destFolderExpanded) {
echo "Copy \"${srcFileExpanded}\" to \"${destFolderExpanded}\""
result = bat( script: """
copy "${srcFileExpanded}" "${destFolderExpanded}"
if errorlevel 1 EXIT /b 1
""",
returnStatus: true
)
if (result != 0) {
return result
}
}
}
return result
}
// Import all eclipse projects in a given folder
def doImportProjects(ds5folder, ds5shellcommand, buildcommand, sourcefolder) {
bat """
"${ds5shellcommand}" ${buildcommand} -no-indexer -importAll "${sourcefolder}"
"""
}
def doBuildAndExportProject(String ds5folder, String ds5shellcommand, String buildcommand, String project, Boolean doexport, variants, String exportfolder, exportpathlist, String destpart) {
echo "Doing build and export project: ${project} : ${doexport}"
for (build_variant in variants) {
retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing)
try {
if (toBuild.contains(build_variant)) {
doDisplayLeftovers()
echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer -build ${project}/${build_variant}\'"
timeout(time: 20, unit: 'MINUTES') {
bat """
"${ds5shellcommand}" ${buildcommand} -no-indexer -build ${project}/${build_variant}
"""
}
result = doCheck(exportpathlist, build_variant)
if (result != 0) {
error "Failed to check files...."
}
echo "Doing export: ${doexport}"
if (doexport) {
result = doExport(exportfolder, exportpathlist, destpart, build_variant)
if (result != 0) {
error "Failed to copy...."
}
}
}
else {
echo "Not building ${build_variant} variant of project: ${project}"
}
} catch (Exception e) {
echo "Exception: ${e.getMessage()}\nBuild failed - Waiting for a while"
sleep(time: 4, unit: 'SECONDS')
error "Build failed..."
}
}
}
}
def doExportOnlyProject(String project, Boolean doexport, variants, String exportfolder, exportpathlist, String destpart) {
echo "Doing export only project: ${project} : ${doexport}"
if (doexport) {
if (!variants) {
result = doExport(exportfolder, exportpathlist, destpart)
if (result != 0) {
error "Failed to copy...."
}
} else {
for (build_variant in variants) {
if (toBuild.contains(build_variant)) {
result = doExport(exportfolder, exportpathlist, destpart, build_variant)
if (result != 0) {
error "Failed to copy...."
}
}
}
}
}
}
// Build and export files
def doBuild(apps, ds5folder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}") {
toBuild = []
if (variantstobuild) {
toBuildUntrimmed = variantstobuild.split(',')
toBuildUntrimmed.each {
toBuild.add(it.trim())
}
}
// The lock is being used because it was being found that the doKillLeftovers was affecting other builds
lock(resource: "eclipsebuild_${env.NODE_NAME}") {
try {
timeout(time: 5, unit: 'MINUTES') {
doImportProjects(ds5folder, ds5shellcommand, buildcommand, sourcefolder)
}
for (appInfo in apps) {
def Boolean build
def Boolean doexport
(build, doexport, project, variants, exportpathlist, destpart) = appInfo
if (build) {
doBuildAndExportProject(ds5folder, ds5shellcommand, buildcommand, project, doexport, variants, exportfolder, exportpathlist, destpart)
}
else {
doExportOnlyProject(project, doexport, variants, exportfolder, exportpathlist, destpart)
}
}
} catch (Exception e) {
// doKillLeftovers()
error "Exception: ${e.getMessage()}\nBuild failed - Killing leftovers"
}
}
}
return this;

View file

@ -227,7 +227,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou
// The lock is being used because it was being found that the doKillLeftovers was affecting other builds
lock(resource: "eclipsebuild_${env.NODE_NAME}") {
try {
timeout(time: 5, unit: 'MINUTES') {
timeout(time: 10, unit: 'MINUTES') {
addToolchainPath(toolchainfolder)
doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder)
}

View file

@ -26,7 +26,9 @@ pipeline {
stage('Copying logic artifacts') {
steps {
script {
importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC_RELEASE_REVISION, LOGICDEST)
def logicoptions = importer.getNewImportOptions()
logicoptions.source(LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_ARTIFACT_RELEASE_REPO, LOGIC_ARTIFACT_RELEASE_REVISION)
importer.artifactimport(meta, logicoptions, LOGICDEST)
}
}
}

View file

@ -0,0 +1,80 @@
pipeline {
agent { label "${AGENT_LABEL}" }
stages {
stage('Initial setup') {
steps {
script {
meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this)
}
}
}
stage('Clean imports') {
steps {
script {
env.DOXYGEN = "doxygen"
cleanfolders = [
DOXYGEN,
]
cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy")
cleaner.clean(cleanfolders)
}
}
}
stage('Import doxygen artifacts') {
steps {
script {
importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
def importoptions = importer.getNewImportOptions()
importoptions.jenkinsjob(BUILD_JOB_WITH_DOXYGEN_ARTIFACTS, BUILD_JOB_WITH_DOXYGEN_ARTIFACTS_SELECTOR)
importoptions.setfilter(ARTIFACT_PATH_TO_DOXYGEN_ROOT + '/**')
doxygenmeta = importer.artifactimportfrombuildjob(meta, importoptions, DOXYGEN)
productname = doxygenmeta.productinfo.productname
buildstampprefix = doxygenmeta.buildstampinfo.buildstampprefix
echo "Doyxgen imported for product: ${productname} and build stamp prefix: ${buildstampprefix}"
}
}
}
stage('Copy doxygen to server') {
steps {
script {
doxygensource = "${DOXYGEN}/${ARTIFACT_PATH_TO_DOXYGEN_ROOT}"
pathonserver = "${ROOT_PATH_ON_SERVER.trim()}/${productname}/${buildstampprefix}"
echo "Copying from ${doxygensource} to ${pathonserver}"
if (isUnix()) {
sh """#!/bin/bash
rm -rfv "${pathonserver}"
if [ \$? -ne 0 ]; then exit 1; fi
mkdir -pv "${pathonserver}"
if [ \$? -ne 0 ]; then exit 1; fi
echo mv -v "${doxygensource}"/ "${pathonserver}"
mv -v "${doxygensource}"/ "${pathonserver}"
if [ \$? -ne 0 ]; then exit 1; fi
"""
} else {
error("This job does not run on Windows at the moment")
}
}
}
}
}
post {
always {
echo 'Post-build'
}
failure {
echo 'Failed build'
}
unstable {
echo 'Unstable build'
}
success {
echo 'Succeeded build'
}
}
}

View file

@ -22,14 +22,16 @@ abstract class GISJsonSerialised implements Serializable {
class ProductInfo extends GISJsonSerialised {
String productname
String subcomponentname
String variant
ProductInfo(script, productname = "", subcomponentname = "") {
ProductInfo(script, productname = "", subcomponentname = "", variant = "") {
super(script)
this.productname = productname
this.subcomponentname = subcomponentname
this.variant = variant
}
void set(String productname, String subcomponentname) {
void set(String productname, String subcomponentname, String variant = '') {
// Make sure the fields can be used in git tags
def int pstatus = 0
if (script.isUnix()) {
@ -51,19 +53,34 @@ class ProductInfo extends GISJsonSerialised {
if (sstatus > 0) {
script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name")
}
if (variant) {
def int vstatus = 0
if (script.isUnix()) {
vstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${variant}\" >/dev/null 2>/dev/null", , returnStatus: true)
}
else {
vstatus = script.bat(script: "@git check-ref-format --normalize \"refs/tags/${variant}\" > NUL 2> NUL", , returnStatus: true)
}
if (vstatus > 0) {
script.error("The subcomponent name: ${variant} needs to be renamed as it cannot be used in a git tag name")
}
}
this.productname = productname
this.subcomponentname = subcomponentname
this.variant = variant
}
void deserialiseJson(rawmeta) {
productname = rawmeta['productname']
subcomponentname = rawmeta['subcomponentname']
variant = rawmeta['variant'] ?: ""
}
LinkedHashMap serialisationdata() {
def data = [
'productname' : productname,
'subcomponentname' : subcomponentname,
'variant' : variant,
]
return data
}
@ -170,6 +187,10 @@ class BuildStampInfo extends GISJsonSerialised {
this.buildstamp = buildstamp
}
void set(BuildStampInfo other) {
set(other.buildstampprefix, other.revision, other.buildstamp)
}
String buildstampcommasep() {
def String buildstampcommasep = buildstamp.replace(".", ",")
return buildstampcommasep
@ -191,7 +212,7 @@ class BuildStampInfo extends GISJsonSerialised {
changerevision(revision)
}
void set(String buildstampprefix, String revision) {
void set(String buildstampprefix, String revision = '0') {
set(buildstampprefix, revision.toInteger())
}

View file

@ -17,6 +17,7 @@ class DebRootDefinition {
String GIS_OPT_SCRIPTS_CONTROL
String GIS_OPT_WWW
String GIS_OPT_OTHER
String GIS_OPT_KERNEL
String GIS_OPT_LOGIC
String GIS_OPT_PRINTHEAD
@ -45,6 +46,7 @@ class DebRootDefinition {
GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control"
GIS_OPT_WWW = "${DEBPATH}/opt/gis/www"
GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other"
GIS_OPT_KERNEL = "${DEBPATH}/opt/gis/kernel"
GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic"
GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead"
@ -70,7 +72,6 @@ def buildPackage(String packageout, String uniquepackageout, DebRootDefinition p
cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}"
sudo chmod -v -R 755 "${packagedefs.DEBPATH}"
sudo chown -v -R root: "${packagedefs.DEBPATH}"
mkdir -v -p "${packageout}"

View file

@ -115,6 +115,19 @@ pipeline {
FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}"
PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix
echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}"
// Convert prefix to regular expression
PACKAGE_VERSION_PREFIX_SPLIT = PACKAGE_VERSION_PREFIX.split("[.]")
PACKAGE_VERSION_REGEX = ''
i = 0
while(true) {
if (i == PACKAGE_VERSION_PREFIX_SPLIT.size() - 1) {
PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i]
break;
}
PACKAGE_VERSION_REGEX += PACKAGE_VERSION_PREFIX_SPLIT[i]
PACKAGE_VERSION_REGEX += '\\.'
i++
}
sh """#!/bin/bash -v
# This errors if the snapshot exists already; Ignore it
aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}"
@ -122,7 +135,7 @@ pipeline {
aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}"
if [ \$? -ne 0 ]; then exit 1; fi
aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_PREFIX}.*)'
aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_REGEX}.*)'
if [ \$? -ne 0 ]; then exit 1; fi
aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}"
if [ \$? -ne 0 ]; then exit 1; fi

View file

@ -66,10 +66,10 @@ pipeline {
!release.toLowerCase().startsWith("faulty") &&
!release.toLowerCase().startsWith("ignore") &&
!release.toLowerCase().startsWith("inmainline")) {
//echo("Showing release: ${release}")
echo("Showing release: ${release}")
releases.add(release)
} else {
//echo("Not showing release: ${release}")
echo("Not showing release: ${release}")
}
}
}

View file

@ -1,3 +1,5 @@
gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy")
markunstable = false
unstablereason = ""
@ -373,7 +375,8 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source
def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, String dryrunpush) {
def buildmeta = buildinfo.meta
def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}"
def uriconverted = gh.uniquereporootname(buildmeta.gitinfo.remoteurl)
def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${uriconverted}"
def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}"
def String uniquefoldertemp = "${tempfolder}/${uniquefolder}"
def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}"

View file

@ -5,7 +5,7 @@ pipeline {
steps {
script {
autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy")
meta = autobuilderscript.startbuild(this, PRODUCT_NAME, SUBCOMPONENT_NAME, RELEASE_REPO)
meta = autobuilderscript.startnonbuild(this)
}
}
}

View file

@ -1,5 +1,5 @@
pipeline {
agent { label "${AGENT_LABEL}" }
agent { label AGENT_LABEL }
stages {
stage('Initial setup') {
steps {
@ -53,7 +53,10 @@ pipeline {
SOURCEDIR = "${WORKSPACE}/source"
configurationhelper.gitCheckoutAndRegister(meta, 'source', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false)
SOURCE_PATH = "${SOURCEDIR}"
env.SRC = "${SOURCE_PATH}/${SOURCE_OFFSET_PATH}"
SRC = "${SOURCE_PATH}"
if (SOURCE_OFFSET_PATH) {
SRC += "/${SOURCE_OFFSET_PATH}"
}
}
}
}
@ -81,20 +84,29 @@ pipeline {
steps {
script {
def PPDFOLDER = "${STATICANALYSIS}/ppd"
dir(PPDFOLDER) {
if (isUnix()) {
sh """#!/bin/bash
set -e
export PMD_JAVA_OPTS=-Xmx65536m
"${PPD}/run.sh" cpd --minimum-tokens 100 --files "${SRC}" --language cpp --format xml --failOnViolation false > cpd.xml
"""
} else {
bat """
set PMD_JAVA_OPTS=-Xmx2048m
"${PPD}/cpd.bat" --minimum-tokens 100 --files "${SRC}" --language cpp --format xml --failOnViolation false > cpd.xml
"""
def EXCLUDE_PARAM = ''
if (EXCLUDE_PATHS) {
EXCLUDE_PARAM = '--exclude "'
def exclude_path_list = EXCLUDE_PATHS.split(',')
for (exclude_path in exclude_path_list) {
EXCLUDE_PARAM += "${SRC}/${exclude_path.trim()},"
}
EXCLUDE_PARAM += '"'
}
if (isUnix()) {
sh """#!/bin/bash
set -ev
mkdir -p "${PPDFOLDER}"
export PMD_JAVA_OPTS=-Xmx65536m
"${PPD}/run.sh" cpd --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml"
"""
} else {
bat """
mkdir -p "${PPDFOLDER}"
set PMD_JAVA_OPTS=-Xmx2048m
"${PPD}/cpd.bat" --minimum-tokens 100 --files "${SRC}" --language cpp --format xml ${EXCLUDE_PARAM} --failOnViolation false > "${PPDFOLDER}/cpd.xml"
"""
}
recordIssues(tools: [
cpd(pattern: "**/cpd.xml", reportEncoding: 'UTF-8'),

View file

@ -17,7 +17,7 @@ def main(argv):
pattern = args.pattern
substitute = args.substitute
print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert))
print("Converting instances of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert))
with open(filetoconvert,'r') as c:
fileread = c.read()