Merge commit 'dea77dfa289efef365b5d1b48262c089832a5fa3'

This commit is contained in:
Dan Hawkins 2022-01-14 20:15:34 +00:00
commit 4ea46e3150
4 changed files with 99 additions and 27 deletions

View file

@ -4,7 +4,7 @@ def combineSubjobArtifacts(meta, importer, subJobs) {
for (subJob in subJobs) { for (subJob in subJobs) {
def subJobObject = subJob[0] def subJobObject = subJob[0]
if (subJobObject != null) { if (subJobObject != null) {
def subJobName = subJobObject.getProjectName() def subJobName = subJobObject.getFullProjectName()
def subJobNumber = subJobObject.getNumber() def subJobNumber = subJobObject.getNumber()
def destFolder = subJob[1] def destFolder = subJob[1]
def boolean incrementversionnumber = subJob[2] def boolean incrementversionnumber = subJob[2]

View file

@ -21,6 +21,8 @@ class ImportOptions {
String m_repo String m_repo
String m_revision String m_revision
boolean m_showdetails
ImportOptions(script, ImportOptions(script,
String sourceselect = 'Jenkins job artifact', String sourceselect = 'Jenkins job artifact',
boolean incrementversionnumber = true, boolean incrementversionnumber = true,
@ -31,7 +33,8 @@ class ImportOptions {
boolean flatten = false, boolean flatten = false,
boolean includemetainfilter = true, boolean includemetainfilter = true,
boolean clean = true, boolean clean = true,
appendtoparentmeta = true) { appendtoparentmeta = true,
showdetails = false) {
m_script = script m_script = script
m_sourceselect = sourceselect.trim() m_sourceselect = sourceselect.trim()
m_incrementversionnumber = incrementversionnumber m_incrementversionnumber = incrementversionnumber
@ -43,6 +46,25 @@ class ImportOptions {
m_includemetainfilter = includemetainfilter m_includemetainfilter = includemetainfilter
m_clean = clean m_clean = clean
m_appendtoparentmeta = appendtoparentmeta m_appendtoparentmeta = appendtoparentmeta
m_showdetails = showdetails
}
String show() {
return """m_incrementversionnumber: ${m_incrementversionnumber}
m_filter: ${m_filter}
m_excludes: ${m_excludes}
m_fingerprint: ${m_fingerprint}
m_nested: ${m_nested}
m_flatten: ${m_flatten}
m_includemetainfilter: ${m_includemetainfilter}
m_clean: ${m_clean}
m_appendtoparentmeta: ${m_appendtoparentmeta}
m_sourceselect: ${m_sourceselect}
m_jobname: ${m_jobname}
m_buildnumberorselector: ${m_buildnumberorselector}
m_repo: ${m_repo}
m_revision: ${m_revision}
"""
} }
void setsourceselect(String sourceselect) { void setsourceselect(String sourceselect) {
@ -56,7 +78,7 @@ class ImportOptions {
m_sourceselect = sourceselect m_sourceselect = sourceselect
break break
default: default:
error("Unknown type of artifact source select: ${m_sourceselect}") m_script.error("Unknown type of artifact source select: ${sourceselect}")
} }
} }
@ -96,7 +118,7 @@ class ImportOptions {
m_appendtoparentmeta = appendtoparentmeta m_appendtoparentmeta = appendtoparentmeta
} }
void jenkinsjob(String jobname, buildnumberorselector) { void jenkinsjob(String jobname, String buildnumberorselector) {
m_jobname = jobname.trim() m_jobname = jobname.trim()
m_buildnumberorselector = [ m_buildnumberorselector = [
$class: 'ParameterizedBuildSelector', $class: 'ParameterizedBuildSelector',
@ -109,7 +131,7 @@ class ImportOptions {
m_jobname = jobname.trim() m_jobname = jobname.trim()
m_buildnumberorselector = [ m_buildnumberorselector = [
$class: 'ParameterizedBuildSelector', $class: 'ParameterizedBuildSelector',
parameterName: "<SpecificBuildSelector plugin='copyartifact@1.41'><buildNumber>${buildnumberorselector}</buildNumber></SpecificBuildSelector>" parameterName: "<SpecificBuildSelector plugin='copyartifact@1.46.2'><buildNumber>${buildnumberorselector}</buildNumber></SpecificBuildSelector>"
] ]
setsourceselect('Jenkins job artifact') setsourceselect('Jenkins job artifact')
} }
@ -125,6 +147,10 @@ class ImportOptions {
releaserepo(repo, revision) releaserepo(repo, revision)
setsourceselect(sourceselect) setsourceselect(sourceselect)
} }
void showdetails(boolean showdetails) {
m_showdetails = showdetails
}
} }
ImportOptions getNewImportOptions() { ImportOptions getNewImportOptions() {
@ -151,7 +177,7 @@ def conditionalClean(ImportOptions importoptions, String destpath) {
} }
} }
def artifactimportfrombuildjob(meta, importoptions, String destpath) { def artifactimportfrombuildjob(meta, ImportOptions importoptions, String destpath) {
if (importoptions.m_sourceselect != 'Jenkins job artifact') { if (importoptions.m_sourceselect != 'Jenkins job artifact') {
error("Import options settings have not been set for: Jenkins job artifact") error("Import options settings have not been set for: Jenkins job artifact")
return return
@ -183,7 +209,7 @@ def rewritemetafileasreleased(meta, String filepath) {
meta.saveJsonSerialised(filepath) meta.saveJsonSerialised(filepath)
} }
def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) { def hierarchyartifactimportfromrepo(meta, ImportOptions importoptions, String destpath) {
if (importoptions.m_sourceselect != 'Release repository') { if (importoptions.m_sourceselect != 'Release repository') {
error("Import options settings have not been set for: Release repository") error("Import options settings have not been set for: Release repository")
return return
@ -200,11 +226,16 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) {
def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) def importedmeta = meta.importrelated(importoptions.m_appendtoparentmeta, destpath, rooturl, rootrevision, gh.getGitInfo(destpath))
relatedimportinfo = importedmeta.getassociated() echo "Getting associated metadata from path: ${destpath}"
echo "Cloning associated repos" def collectedmetalist = importedmeta.getassociated(destpath)
for(relatedmetainfo in relatedimportinfo) { for(def collectedmeta in collectedmetalist) {
def relatedmeta = relatedmetainfo.meta echo "Collection meta item: ${collectedmeta.metainfo.componentversionuniqueidentifier()} offset path: ${collectedmeta.accumulatedpath}"
def relatedoffsetpath = relatedmetainfo.offsetpath }
echo "Cloning associated repos inside path: ${destpath}"
for(def collectedmeta in collectedmetalist) {
def relatedmeta = collectedmeta.metainfo
def relatedoffsetpath = collectedmeta.accumulatedpath
def cloneurl = relatedmeta.releaseinfo.releaserepoURL def cloneurl = relatedmeta.releaseinfo.releaserepoURL
def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonerevision = "refs/tags/${relatedmeta.releasetag()}"
def clonepath = "${destpath}/${relatedoffsetpath}" def clonepath = "${destpath}/${relatedoffsetpath}"
@ -214,7 +245,7 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) {
gh.stripgithistory(clonepath) gh.stripgithistory(clonepath)
def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" def metafilepath = "${clonepath}${relatedmeta.metafilepath()}"
echo "Marking related meta as released and rewriting meta file: ${metafilepath}" echo "Marking related meta as released and rewriting meta file: ${metafilepath}"
rewritemetafileasreleased(relatedmeta, metafilepath) rewritemetafileasreleased(relatedmeta, metafilepath)
} }
@ -229,7 +260,7 @@ def hierarchyartifactimportfromrepo(meta, importoptions, String destpath) {
return importedmeta return importedmeta
} }
def artifactimportfromrepo(meta, importoptions, String destpath) { def artifactimportfromrepo(meta, ImportOptions importoptions, String destpath) {
if (importoptions.m_sourceselect != 'Release repository') { if (importoptions.m_sourceselect != 'Release repository') {
error("Import options settings have not been set for: Release repository") error("Import options settings have not been set for: Release repository")
return return
@ -257,7 +288,10 @@ def artifactimportfromrepo(meta, importoptions, String destpath) {
return importedmeta return importedmeta
} }
def artifactimport(meta, importoptions, String destpath) { def artifactimport(meta, ImportOptions importoptions, String destpath) {
if (importoptions.m_showdetails) {
echo "Importing artifacts:\nImport options:\n${importoptions.show()}\nto: ${destpath}"
}
switch (importoptions.m_sourceselect) { switch (importoptions.m_sourceselect) {
case 'Jenkins job artifact': case 'Jenkins job artifact':
case 'Jenkins job artefact': case 'Jenkins job artefact':
@ -342,7 +376,13 @@ def artifactimport(meta,
} }
// Import the artifacts of a named child component found below parent meta's hierarchy // Import the artifacts of a named child component found below parent meta's hierarchy
def artifactimport(meta, parentmeta, String parentmetapath, String childproduct, String childsubcomponent, importoptions, String destinationfolder) { def artifactimport(meta,
parentmeta,
String parentmetapath,
String childproduct,
String childsubcomponent,
ImportOptions importoptions,
String destinationfolder) {
// Import from parent meta // Import from parent meta
def childmeta = meta.getNewMetaObject() def childmeta = meta.getNewMetaObject()
def childmeta2 def childmeta2
@ -351,7 +391,7 @@ def artifactimport(meta, parentmeta, String parentmetapath, String childproduct,
def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent) def findresults = parentmeta.findAssociatedMeta(parentmetapath, childproduct, childsubcomponent)
if (findresults.matchfound) { if (findresults.matchfound) {
childmeta = findresults.matchedmetaitem.metainfo childmeta = findresults.matchedmetaitem.metainfo
if (childmeta.releaseinfo.released) { if (importoptions.m_sourceselect == 'Release repository' || childmeta.releaseinfo.released) {
importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag()) importoptions.releaserepo(childmeta.releaseinfo.releaserepoURL, childmeta.releasetag())
} else { } else {
importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber) importoptions.jenkinsjob(childmeta.jenkinsbuildinfo.buildjobname, childmeta.jenkinsbuildinfo.buildnumber)

View file

@ -23,17 +23,32 @@ def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, Strin
} }
// Given details about artifacts to import and details of installers to build, import artifacts that are required // Given details about artifacts to import and details of installers to build, import artifacts that are required
def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) { def Map artifactsImporter(meta, importer, toBuild, artifactDetails, installerDetails, boolean showdetails = false) {
artifactFetched = [:] artifactFetched = [:]
if (showdetails) {
echo ("toBuild:\n${toBuild}\nartifactDetails:\n${artifactDetails}\ninstallerDetails:\n${installerDetails}")
}
def importoptions = importer.getNewImportOptions() def importoptions = importer.getNewImportOptions()
importoptions.showdetails(showdetails)
for (installer in toBuild) { for (installer in toBuild) {
def artifactKeys = installerDetails.get(installer)[1] def installerinfo = installerDetails.get(installer)
if (showdetails) {
echo ("installer:\n${installer}\ninstallerinfo:\n${installerinfo}")
}
def artifactKeys = installerinfo[1]
if (artifactKeys instanceof java.lang.String) { if (artifactKeys instanceof java.lang.String) {
artifactKeys = [ artifactKeys ] artifactKeys = [ artifactKeys ]
} }
if (showdetails) {
echo ("artifactKeys:\n${artifactKeys}")
}
for (artifactKey in artifactKeys) { for (artifactKey in artifactKeys) {
(sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) artifactDetail = artifactDetails.get(artifactKey)
if (showdetails) {
echo ("artifactKey: ${artifactKey}\nartifactDetail:\n${artifactDetail}")
}
(sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetail
if (artifactFetched.get(artifactKey) == null) { if (artifactFetched.get(artifactKey) == null) {
importoptions.jenkinsjob(sourceProject, sourceselector) importoptions.jenkinsjob(sourceProject, sourceselector)
importoptions.setincrementversionnumber(incrementversionnumber) importoptions.setincrementversionnumber(incrementversionnumber)
@ -45,6 +60,9 @@ def Map artifactsImporter(meta, importer, artifactDetails, installerDetails) {
} }
} }
} }
if (showdetails) {
echo ("artifactFetched:\n${artifactFetched}")
}
return artifactFetched return artifactFetched
} }
@ -93,6 +111,7 @@ def checkISMfilebackend(String xml) {
def checkISMfile(String ISMfile) { def checkISMfile(String ISMfile) {
echo "Checking ISM file: ${ISMfile}" echo "Checking ISM file: ${ISMfile}"
def xml = readFile(ISMfile) def xml = readFile(ISMfile)
//echo "ISM file xml: ${xml}"
(exitcode, message) = checkISMfilebackend(xml) (exitcode, message) = checkISMfilebackend(xml)
echo message echo message
if (exitcode > 0) { if (exitcode > 0) {

View file

@ -608,6 +608,7 @@ class MetaInfo extends GISJsonSerialised {
abstract void deserialiseJson(rawmeta) abstract void deserialiseJson(rawmeta)
abstract LinkedHashMap serialisationdata() abstract LinkedHashMap serialisationdata()
abstract String getrelatedid() abstract String getrelatedid()
abstract void markreleased()
} }
class RelatedMetaInfo_v0_1 extends RelatedMetaInfo { class RelatedMetaInfo_v0_1 extends RelatedMetaInfo {
@ -642,6 +643,10 @@ class MetaInfo extends GISJsonSerialised {
String getrelatedid() { String getrelatedid() {
return uniqueid return uniqueid
} }
void markreleased() {
meta.releaseinfo.markreleased()
}
} }
class RelatedMetaInfo_v0_2 extends RelatedMetaInfo { class RelatedMetaInfo_v0_2 extends RelatedMetaInfo {
@ -674,6 +679,10 @@ class MetaInfo extends GISJsonSerialised {
String getrelatedid() { String getrelatedid() {
return uniqueid return uniqueid
} }
void markreleased() {
// Do nothing
}
} }
private ProductInfo productinfo private ProductInfo productinfo
@ -701,15 +710,11 @@ class MetaInfo extends GISJsonSerialised {
associatedmeta = new ArrayList<RelatedMetaInfo>() associatedmeta = new ArrayList<RelatedMetaInfo>()
} }
List getassociated() {
return associatedmeta
}
void markreleased() { void markreleased() {
script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released")
releaseinfo.markreleased() releaseinfo.markreleased()
for (associated in associatedmeta) { for (associated in associatedmeta) {
associated.meta.releaseinfo.markreleased() associated.markreleased()
} }
} }
@ -1079,6 +1084,7 @@ class MetaInfo extends GISJsonSerialised {
return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ] return [ 'metacollection' : ii.metacollection, 'metamap' : ii.metamap, 'comment': ii.comment ]
} }
// Walks all layers of associated data only
def allrelatedmeta(String metapath = '') { def allrelatedmeta(String metapath = '') {
def ArrayList<CollectedMeta> metacollection = new ArrayList<CollectedMeta>() def ArrayList<CollectedMeta> metacollection = new ArrayList<CollectedMeta>()
def Map<String, CollectedMeta> metamap = new LinkedHashMap<String, CollectedMeta>() def Map<String, CollectedMeta> metamap = new LinkedHashMap<String, CollectedMeta>()
@ -1088,10 +1094,17 @@ class MetaInfo extends GISJsonSerialised {
ArrayList<CollectedMeta> releasewalker(int maxdepth, String startpath = '', String startmetapath = '') { ArrayList<CollectedMeta> releasewalker(int maxdepth, String startpath = '', String startmetapath = '') {
def ArrayList<CollectedMeta> metacollection = new ArrayList<CollectedMeta>() def ArrayList<CollectedMeta> metacollection = new ArrayList<CollectedMeta>()
def Map<String, CollectedMeta> metamap = new LinkedHashMap<String, CollectedMeta>() def Map<String, CollectedMeta> metamap = new LinkedHashMap<String, CollectedMeta>()
def results = walkrelated(new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth)) def IterationInfo ii = new IterationInfo(metacollection, metamap, 'meta', 0, startpath, startmetapath, true, maxdepth)
script.echo("Release walker iteration info at start:\n${ii.show()}")
def results = walkrelated(ii)
return results.metacollection return results.metacollection
} }
// Walks 1 layer of associated data only
def getassociated(String rootpath) {
return releasewalker(2, '', "${rootpath}/${METAFOLDERNAME}") // Max depth 2 (including itself)
}
Map<String, CollectedMeta> putwithshortestoffsetpath(Map<String, CollectedMeta> releasemetacollection, String uniquestring, CollectedMeta metaitem) { Map<String, CollectedMeta> putwithshortestoffsetpath(Map<String, CollectedMeta> releasemetacollection, String uniquestring, CollectedMeta metaitem) {
if (releasemetacollection.containsKey(uniquestring)) { if (releasemetacollection.containsKey(uniquestring)) {
def existingentry = releasemetacollection[uniquestring] def existingentry = releasemetacollection[uniquestring]