diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 30b7086..1864ba9 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -9,7 +9,7 @@ def registerPipelineScriptConfiguration(meta) { } def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { - gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 1d6885f..11b3733 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -1,13 +1,743 @@ +import groovy.json.JsonSlurper + +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" + +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" + +reportfolder = "${WORKSPACE}/reports" + +class JiraTicketInfo { + String jiranumber + String uri + + JiraTicketInfo() { + jiranumber = '' + uri = '' + } + + def String dump() { + return "jiranumber: ${jiranumber} uri: ${uri}" + } +} + +class DifferenceRecord { + String productname + String subcomponentname + String jirakey + + String rawdifferencetext + String sha + String type + String mergeinfo + List jiraticketinfo + String comment + + DifferenceRecord() { + productname = "" + subcomponentname = "" + jirakey = "" + + rawdifferencetext = "" + sha = "" + type = "" + mergeinfo = "" + jiraticketinfo = [] + comment = "" + } + + def String dump() { + String jiraticketstring = '' + for (j in jiraticketinfo) { + jiraticketstring += j.dump() + " " + } + return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" + } +} + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } + dir(reportfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def showmetaitems(metaitems) { + String info = '' + for (key in metaitems.keySet()) { + def relatedmetainfo = metaitems[key] + def offsetpath = relatedmetainfo.offsetpath + info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" + } + echo info +} + +def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { + boolean checkedout = false + def String sourcerepo = buildmeta.gitinfo.remoteurl + if (sourcerepo?.trim()) { + echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" + if (sourcerepo) { + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } + } + checkedout = true + } + return checkedout +} + +def String getcomponent(meta) { + def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" + + boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) + + return [checkedout, uniquefolderforsourcerepo] +} + +def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { + def differencerecords = [] + def String releaseexpression + if (tripledot) { + releaseexpression = earlyrevision + "..." + laterrevision + } else { + releaseexpression = earlyrevision + ".." + laterrevision + } + def String differencetext + dir(gitrootfolder) { + differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() + } + return differencetext +} + +def extractsha(String line) { + def pattern = ~"^(\\p{XDigit}*) (.*)" + def match = (line =~ pattern) + match.find() + + def sha = match.group(1) + def restofline = match.group(2) + + //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" + + return [sha.trim(), restofline.trim()] +} + +def extractmerges(String line) { + def pattern1 = "^(.*)(Merge branch .*)" + def match1 = (line =~ pattern1) + if (match1.find()) { + return [true, match1.group(2)] + } + + def pattern2 = "^(.*)(Merge remote-tracking branch .*)" + def match2 = (line =~ pattern2) + if (match2.find()) { + return [true, match2.group(2)] + } + + def pattern3 = "^(.*)(Merge commit .*)" + def match3 = (line =~ pattern3) + if (match3.find()) { + return [true, match3.group(2)] + } + + def pattern4 = "^(.*)(Add .* from commit .*)" + def match4 = (line =~ pattern4) + if (match4.find()) { + return [true, match4.group(2)] + } + + return [false, line] +} + +def jiraticketexists(String jiraticket) { + def String URI = "${JIRA_URI}/browse/${jiraticket}" + def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" + def get = new URL(RestfulURI).openConnection(); + get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) + get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) + get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) + get.setRequestProperty( 'Accept', 'application/json' ) + def getRC = get.getResponseCode(); + + def String redirectedjiraticket = jiraticket + if(!getRC.equals(200)) { + return [false, '', jiraticket] + } else { + def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) + redirectedjiraticket = jsonresponse['key'] + } + return [true, URI, redirectedjiraticket] +} + +def extractjiraticketinfo(String line) { + def List jiratickets = [] + boolean exists = false + String URI = '' + + while (true) { + line = line.trim() + def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line + def commamatch = (line =~ commapattern) + if (commamatch.find()) { + line = commamatch.group(1).trim() + } + def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 + def match1 = (line =~ pattern1) + if (match1.find()) { + jiraticket = match1.group(1) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match1.group(2) + } else { + break + } + } else { + def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 + def match2 = (line =~ pattern2) + if (match2.find()) { + jiraticket = match2.group(1) + "-" + match2.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match2.group(3) + } else { + break + } + } else { + def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 + def match3 = (line =~ pattern3) + if (match3.find()) { + jiraticket = match3.group(1) + "-" + match3.group(2) + (exists, URI, jiraticket) = jiraticketexists(jiraticket) + if (exists) { + def JiraTicketInfo j = new JiraTicketInfo() + j.jiranumber = jiraticket + j.uri = URI + jiratickets.add(j) + line = match3.group(3) + } else { + break + } + } else { + break + } + } + } + } + boolean result = jiratickets.size() > 0 + return [result, jiratickets, line] +} + +def extractreverts(String line) { + boolean result = true + + def pattern1 = '^(.*)Revert \"(.*)\"' + def match1 = (line =~ pattern1) + if (match1.find()) { + def line2 = match1.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + def pattern2 = '^(.*)Partial Revert \"(.*)\"' + def match2 = (line =~ pattern2) + if (match2.find()) { + def line2 = match2.group(2) + (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) + return [true, originaljiraticketinfo, originalcomment] + } + + return [false, [], line] +} + +def DifferenceRecord fillrecord(meta, line) { + def DifferenceRecord dr = new DifferenceRecord() + + dr.productname = meta.productinfo.productname + dr.subcomponentname = meta.productinfo.subcomponentname + dr.jirakey = meta.productinfo.jirakey + + dr.rawdifferencetext = line + + // Example of raw text to process: + // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py + + (sha, line2) = extractsha(line) + dr.sha = sha + + (result, mergeinfo) = extractmerges(line2) + if (result) { + dr.type = "Merge" + dr.mergeinfo = mergeinfo + } else { + (result, jiraticketinfo, comment) = extractreverts(line2) + if (result) { + dr.type = "Revert" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) + if (result) { + dr.type = "Ticketed" + dr.jiraticketinfo = jiraticketinfo + dr.comment = comment + } else { + dr.type = "Unticketed" + dr.comment = line2 + } + } + } + + return dr +} + +def List processtext(meta, String text) { + def List differencerecords = [] + + text.split('\n').each { + def line = it + if (line?.trim()) { + def DifferenceRecord dr = fillrecord(meta, line.trim()) + differencerecords.add(dr) + //echo "${dr.dump()}" + } + } + return differencerecords +} + +def getdifferencerecords(earlymeta, latermeta, String sourcepath) { + boolean result = true; + def List differencerecords = [] + + def earlygitinfo = earlymeta.gitinfo + def latergitinfo = latermeta.gitinfo + + if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { + def earlyrevision = earlygitinfo.gitrevision + def laterrevision = latergitinfo.gitrevision + echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" + summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") + def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) + differencerecords = processtext(earlymeta, differencetext) + if (earlyrevision == laterrevision) { + summaryappend(" No change") + } + } else { + summaryappend("The early and later git urls don't match") + echo "The early and later git urls don't match" + result = false; + setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") + } + + return [result, differencerecords] +} + +def deletedcomponent(earlymetaitems, key) { + echo "Component ${key} has been deleted in later version" + summaryappend("Deleted ") +} + +def addedcomponent(latermetaitems, key) { + echo "Component ${key} has been added in later version" + summaryappend("Added ") +} + +class MergeReportRecord { + String comment + int duplicates + + MergeReportRecord() { + comment = '' + duplicates = 0 + } +} + +def mergereport(String reportsubfolder, LinkedHashMap merges, boolean iscomponent = true) { + dir(reportsubfolder) { + def String mergelist = "Merge list:\n" + def mergessorted = merges.sort() + for (amerge in mergessorted) { + def MergeReportRecord m = amerge.value + mergelist += "${m.comment} (${m.duplicates})\n" + } + writeFile(file: "mergelist.txt", text: mergelist) + } +} + +class CommentsRecord { + String comment + int duplicates + + CommentsRecord() { + comment = '' + duplicates = 0 + } +} + +class RevertsReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + RevertsReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def revertsreport(String reportsubfolder, LinkedHashMap reverts, boolean iscomponent = true) { + dir(reportsubfolder) { + def String revertlist = "Revert list:\n" + def revertssorted = reverts.sort() + for (arevert in revertssorted) { + def RevertsReportRecord r = arevert.value + revertlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + revertlist += "${c.comment} (${c.duplicates})\n" + } + revertlist += "\n" + } + writeFile(file: "revertlist.txt", text: revertlist) + } +} + +class TicketedReportRecord { + String jiraticket + String ticketURI + LinkedHashMap commentscollection + + TicketedReportRecord() { + jiraticket = '' + ticketURI = '' + commentscollection = new LinkedHashMap() + } +} + +def ticketedreport(String reportsubfolder, LinkedHashMap ticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String ticketedlist = "Ticketed list:\n" + def ticketedsorted = ticketed.sort() + for (aticketed in ticketedsorted) { + def TicketedReportRecord r = aticketed.value + ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" + def commentscollection = r.commentscollection + def sortedcommentscollection = commentscollection.sort() + for (commentrecord in sortedcommentscollection) { + def CommentsRecord c = commentrecord.value + ticketedlist += "${c.comment} (${c.duplicates})\n" + } + ticketedlist += "\n" + } + writeFile(file: "ticketedlist.txt", text: ticketedlist) + } +} + +class UnticketedReportRecord { + String comment + int duplicates + + UnticketedReportRecord() { + comment = '' + duplicates = 0 + } +} + +def unticketedreport(String reportsubfolder, LinkedHashMap unticketed, boolean iscomponent = true) { + dir(reportsubfolder) { + def String unticketedlist = "Unticketed list:\n" + def unticketedsorted = unticketed.sort() + for (anunticketed in unticketedsorted) { + UnticketedReportRecord u = anunticketed.value + unticketedlist += "${u.comment} (${u.duplicates})\n" + } + writeFile(file: "unticketedlist.txt", text: unticketedlist) + } +} + +class TicketsReportRecord { + String jiraticket + String ticketURI + int duplicates + + TicketsReportRecord() { + jiraticket = '' + ticketURI = '' + duplicates = 0 + } +} + +def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { + dir(reportsubfolder) { + String ticketlist = "Ticket list:\n" + ticketssorted = tickets.sort() + for (ticketnumber in ticketssorted) { + TicketsReportRecord t = ticketnumber.value + ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" + } + writeFile(file: "ticketlist.txt", text: ticketlist) + } +} + +def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { + dir(reportsubfolder) { + String recordlist = "" + for (dr in differencerecords) { + recordlist += dr.dump() + "\n\n" + } + writeFile(file: "recordlist.txt", text: recordlist) + } +} + +def boolean report(meta, differencerecords, boolean iscomponent = true) { + def Map ticketed = [:] + def Map merges = [:] + def Map reverts = [:] + def Map unticketed = [:] + def Map tickets = [:] + + for (dr in differencerecords) { + if (dr.type == "Merge") { + def MergeReportRecord m + if (!merges.containsKey(dr.mergeinfo)) { + m = new MergeReportRecord() + m.comment = dr.mergeinfo + m.duplicates = 1 + } else { + m = merges[dr.mergeinfo] + m.duplicates += 1 + } + merges[dr.mergeinfo] = m + } + if (dr.type == "Revert") { + def RevertsReportRecord r + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!reverts.containsKey(j.jiranumber)) { + r = new RevertsReportRecord() + r.jiraticket = j.jiranumber + r.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + r = reverts[j.jiranumber] + def CommentsRecord c + if (!r.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + r.commentscollection[dr.comment] = c + } else { + c = r.commentscollection[dr.comment] + c.duplicates += 1 + } + r.commentscollection[dr.comment] = c + } + reverts[j.jiranumber] = r + def TicketsReportRecord t + if (!tickets.containsKey(j.jiranumber)) { + t = new TicketsReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + t.duplicates = 1 + } else { + t = tickets[j.jiranumber] + t.duplicates += 1 + } + tickets[j.jiranumber] = t + } + } + if (dr.type == "Ticketed") { + def TicketedReportRecord t + for (ticket in dr.jiraticketinfo) { + JiraTicketInfo j = ticket + if (!ticketed.containsKey(j.jiranumber)) { + t = new TicketedReportRecord() + t.jiraticket = j.jiranumber + t.ticketURI = j.uri + CommentsRecord c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + t = ticketed[j.jiranumber] + def CommentsRecord c + if (!t.commentscollection.containsKey(dr.comment)) { + c = new CommentsRecord() + c.comment = dr.comment + c.duplicates = 1 + t.commentscollection[dr.comment] = c + } else { + c = t.commentscollection[dr.comment] + c.duplicates += 1 + } + t.commentscollection[dr.comment] = c + } + ticketed[j.jiranumber] = t + def TicketsReportRecord ti + if (!tickets.containsKey(j.jiranumber)) { + ti = new TicketsReportRecord() + ti.jiraticket = j.jiranumber + ti.ticketURI = j.uri + ti.duplicates = 1 + } else { + ti = tickets[j.jiranumber] + ti.duplicates += 1 + } + tickets[j.jiranumber] = ti + } + } + if (dr.type == "Unticketed") { + UnticketedReportRecord u + if (!unticketed.containsKey(dr.comment)) { + u = new UnticketedReportRecord() + u.comment = dr.comment + u.duplicates = 1 + } else { + u = unticketed[dr.comment] + u.duplicates += 1 + } + unticketed[dr.comment] = u + } + } + String reportsubfolder = "" + if (iscomponent) { + reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" + } else { + reportsubfolder = "${reportfolder}/Combined" + } + mergereport(reportsubfolder, merges, iscomponent) + revertsreport(reportsubfolder, reverts, iscomponent) + ticketedreport(reportsubfolder, ticketed, iscomponent) + unticketedreport(reportsubfolder, unticketed, iscomponent) + ticketsreport(reportsubfolder, tickets, iscomponent) + differencerecordsreport(reportsubfolder, differencerecords, iscomponent) +} + +def compare(earlymetaitems, latermetaitems, key) { + boolean result = true; + echo "Collecting changes between early and later versions of component: ${key}" + + def earlymeta = earlymetaitems[key].meta + def latermeta = latermetaitems[key].meta + + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") + + def boolean repoavailable + def String uniquefolderforsourcerepo + (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) + + List differencerecords = [] + if (repoavailable) { + (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) + + summaryappend(" Items: ${differencerecords.size()} ") + report(earlymeta, differencerecords) + } else { + summaryappend(" No source repo ") + } + summaryappend("\n") + + return [result, differencerecords]; +} def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { + def List differencerecords = [] + def int md = maxdepth.toInteger() + resetfolders() + preconditions() - echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" + echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" - def String summarystring = "No summary" - def boolean markunstable = true - def unstablereason = "No compare implementation yet" + def earlymeta = meta.importrelated(earlymetafolder) + def latermeta = meta.importrelated(latermetafolder) + def earlymetaitems = earlymeta.releasewalkerunique(md) + def latermetaitems = latermeta.releasewalkerunique(md) + + echo "Early items:" + showmetaitems(earlymetaitems) + + echo "Later items:" + showmetaitems(latermetaitems) + + for (key in earlymetaitems.keySet()) { + if (latermetaitems.containsKey(key)) { + (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) + differencerecords = differencerecords + componentdifferencerecords + } else { + summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") + deletedcomponent(earlymetaitems, key) + summaryappend("\n") + } + } + for (key in latermetaitems.keySet()) { + if (!earlymetaitems.containsKey(key)) { + summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") + addedcomponent(latermetaitems, key) + summaryappend("\n") + } + } + report(meta, differencerecords, false) return [summarystring, markunstable, unstablereason] } diff --git a/differences/metahierarchy.groovy b/differences/metahierarchy.groovy index cd6738d..4bcdcdc 100644 --- a/differences/metahierarchy.groovy +++ b/differences/metahierarchy.groovy @@ -13,33 +13,32 @@ pipeline { steps { script { SUMMARYDIR = "${WORKSPACE}/summary" + REPORTDIR = "${WORKSPACE}/reports" + EARLYROOTDIR = 'early' + EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" + LATERROOTDIR = 'later' + LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" + cleanfolders = [ "${WORKSPACE}/build_artifacts", - "${WORKSPACE}/early", - "${WORKSPACE}/later", + "${EARLYDIR}", + "${LATERDIR}", "${SUMMARYDIR}", + "${REPORTDIR}", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) } } } - stage('Checkout top level meta') { + stage('Obtain top level meta') { steps { script { - configurationhelper = autobuilder.configurationhelper() + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim() - - EARLYREVISION = RELEASE_EARLY_REVISION.trim() - EARLYROOTDIR = 'early' - EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}" - configurationhelper.gitCheckoutAndRegister(meta, EARLYROOTDIR, SOURCEURL, EARLYREVISION, EARLYDIR, EARLYROOTDIR, 0, false) - - LATERREVISION = RELEASE_LATER_REVISION.trim() - LATERROOTDIR = 'later' - LATERDIR = "${WORKSPACE}/${LATERROOTDIR}" - configurationhelper.gitCheckoutAndRegister(meta, LATERROOTDIR, SOURCEURL, LATERREVISION, LATERDIR, LATERROOTDIR, 0, false) + importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**') + importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**') } } } @@ -59,7 +58,7 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts() - archiveArtifacts "summary/**,${artifactlist}" + archiveArtifacts "summary/**,reports/**,${artifactlist}" } } } diff --git a/import/importer.groovy b/import/importer.groovy index 09eb337..ba5eb0b 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -17,7 +17,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, bo return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -42,11 +42,14 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - gh.gitCheckout(rooturl, rootrevision, destpath) + + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) @@ -59,8 +62,11 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d def clonerevision = "refs/tags/${relatedmeta.releasetag()}" def clonepath = "${destpath}/${relatedoffsetpath}" echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}" - gh.gitCheckout(cloneurl, clonerevision, clonepath) + + gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, []) + gh.stripgithistory(clonepath) + def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}" echo "Marking related meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(relatedmeta, metafilepath) @@ -76,7 +82,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d return importedmeta } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -90,18 +96,21 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) - + if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) } return importedmeta } -def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { switch (source) { case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) case "Release repository": + if (nested) { + return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) default: error("Unknown type of artifact source: ${source}") diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..18ec404 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -8,7 +8,6 @@ summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos" -tempfolder = "${WORKSPACE}/temp" def resetfolders() { dir(summaryfolder) { @@ -20,9 +19,6 @@ def resetfolders() { dir(filteredreleaserepofolder) { deleteDir() } - dir(tempfolder) { - deleteDir() - } } def productuniquefolder(releasemeta) { @@ -61,8 +57,7 @@ def guessreporootsuffix(componentURL) { } gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) { - // Copy relevant artifacts to temp folder +def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) { summaryappend("Artifacts: ") def releasemeta = releaseinfo.meta @@ -124,11 +119,10 @@ def copydirectory(File dirFrom, File dirTo) { def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) { def releasemeta = releaseinfo.meta def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta) - def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta) summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") - getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish) + getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish) summaryappend("\n") } diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 811d072..3ea97ec 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -249,13 +249,16 @@ def gitsparserestore(String targetDir) { def List convertSparseItemsString(String itemstopublish) { def List sparsepaths = [] - def String[] items = itemstopublish.split(',') - def String allpaths = "" - for(String item : items) { - sparsepaths.add(item) - allpaths += "${item}\n" + boolean nonempty = itemstopublish?.trim() + if (nonempty) { + def String[] items = itemstopublish.trim().split(',') + def String allpaths = "" + for(String item : items) { + sparsepaths.add(item) + allpaths += "${item}\n" + } + echo "Selected item to publish:\n${allpaths}" } - echo "Selected item to publish:\n${allpaths}" return sparsepaths }