import groovy.json.JsonSlurper markunstable = false unstablereason = "" summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" reportfolder = "${WORKSPACE}/reports" class JiraTicketInfo { String jiranumber String uri JiraTicketInfo() { jiranumber = '' uri = '' } def String dump() { return "jiranumber: ${jiranumber} uri: ${uri}" } } class DifferenceRecord { String productname String subcomponentname String rawdifferencetext String sha String type String mergeinfo String squashinfo List jiraticketinfo String comment DifferenceRecord() { productname = "" subcomponentname = "" rawdifferencetext = "" sha = "" type = "" mergeinfo = "" squashinfo = "" jiraticketinfo = [] comment = "" } def String dump() { String jiraticketstring = '' for (j in jiraticketinfo) { jiraticketstring += j.dump() + " " } return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}" } } def resetfolders() { dir(summaryfolder) { deleteDir() } dir(tempfolder) { deleteDir() } dir(reportfolder) { deleteDir() } } def preconditions() { markunstable = false unstablereason = "" summarystring = "" writeFile(file: summaryfile, text: "Release summary: \n") } def setunstable(String reason) { markunstable = true unstablereason += reason + "\n" echo "Marked build as unstable: Reason: ${reason}" } def summaryappend(String summary) { def String currentsummarycontents = readFile(file: summaryfile) writeFile(file: summaryfile, text: currentsummarycontents + summary) summarystring += summary } def showmetaitems(metaitems) { String info = '' for (key in metaitems.keySet()) { def relatedmetainfo = metaitems[key] def offsetpath = relatedmetainfo.offsetpath info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n" } echo info } def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) { boolean checkedout = false def String sourcerepo = buildmeta.gitinfo.remoteurl if (sourcerepo?.trim()) { echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}" if (sourcerepo) { // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component if (fileExists(checkoutfolder)) { dir(checkoutfolder) { sh(""" git reset --hard git clean -fdx git pull """ ) } } else { dir(sourcerepofolder) { sh("git clone ${sourcerepo} ${checkoutfolder}") } } } checkedout = true } return checkedout } def String getcomponent(meta) { def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo) return [checkedout, uniquefolderforsourcerepo] } def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) { def String releaseexpression if (tripledot) { releaseexpression = earlyrevision + "..." + laterrevision } else { releaseexpression = earlyrevision + ".." + laterrevision } def String differencetext dir(gitrootfolder) { differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim() } return differencetext } def extractsha(String line) { def pattern = ~"^(\\p{XDigit}*) (.*)" def match = (line =~ pattern) match.find() def sha = match.group(1) def restofline = match.group(2) //echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}" return [sha.trim(), restofline.trim()] } def extractsquashes(String line) { def pattern1 = "^(.*)(Squashed .*)" def match1 = (line =~ pattern1) if (match1.find()) { return [true, match1.group(2)] } return [false, line] } def extractmerges(String line) { def pattern1 = "^(.*)(Merge branch .*)" def match1 = (line =~ pattern1) if (match1.find()) { return [true, match1.group(2)] } def pattern2 = "^(.*)(Merge remote-tracking branch .*)" def match2 = (line =~ pattern2) if (match2.find()) { return [true, match2.group(2)] } def pattern3 = "^(.*)(Merge commit .*)" def match3 = (line =~ pattern3) if (match3.find()) { return [true, match3.group(2)] } def pattern4 = "^(.*)(Add .* from commit .*)" def match4 = (line =~ pattern4) if (match4.find()) { return [true, match4.group(2)] } return [false, line] } def jiraticketexists(String jiraticket) { def String URI = "${JIRA_URI}/browse/${jiraticket}" def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary" def get = new URL(RestfulURI).openConnection(); get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" ) get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" ) get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' ) get.setRequestProperty( 'Accept', 'application/json' ) def getRC = get.getResponseCode(); def String redirectedjiraticket = jiraticket if(!getRC.equals(200)) { return [false, '', jiraticket] } else { def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8')) redirectedjiraticket = jsonresponse['key'] } return [true, URI, redirectedjiraticket] } def extractjiraticketinfo(String line) { def List jiratickets = [] boolean exists = false String URI = '' while (true) { line = line.trim() def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line def commamatch = (line =~ commapattern) if (commamatch.find()) { line = commamatch.group(1).trim() } def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234 def match1 = (line =~ pattern1) if (match1.find()) { jiraticket = match1.group(1) (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) line = match1.group(2) } else { break } } else { def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234 def match2 = (line =~ pattern2) if (match2.find()) { jiraticket = match2.group(1) + "-" + match2.group(2) (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) line = match2.group(3) } else { break } } else { def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234 def match3 = (line =~ pattern3) if (match3.find()) { jiraticket = match3.group(1) + "-" + match3.group(2) (exists, URI, jiraticket) = jiraticketexists(jiraticket) if (exists) { def JiraTicketInfo j = new JiraTicketInfo() j.jiranumber = jiraticket j.uri = URI jiratickets.add(j) line = match3.group(3) } else { break } } else { break } } } } boolean result = jiratickets.size() > 0 return [result, jiratickets, line] } def extractreverts(String line) { boolean result = true def pattern1 = '^(.*)Revert \"(.*)\"' def match1 = (line =~ pattern1) if (match1.find()) { def line2 = match1.group(2) (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) return [true, originaljiraticketinfo, originalcomment] } def pattern2 = '^(.*)Partial Revert \"(.*)\"' def match2 = (line =~ pattern2) if (match2.find()) { def line2 = match2.group(2) (result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2) return [true, originaljiraticketinfo, originalcomment] } return [false, [], line] } def getgitlogdifferencesfromsquash(String gitrootfolder, String squashedrevision) { def String differencetext dir(gitrootfolder) { differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${squashedrevision}").trim() } return differencetext } def List fillsquashrecords(String sourcepath, String sha) { def List differencerecords = [] String squashloglines = getgitlogdifferencesfromsquash(sourcepath, sha) int count = 0 squashloglines.split('\n').each { if (count > 0) { // Ignore first line def line = it if (line?.trim()) { def List drs = fillrecord(meta, line.trim(), sourcepath) differencerecords.addAll(drs) //echo "${dr.dump()}" } } count += 1 } return differencerecords } def List fillrecord(meta, line, String sourcepath) { def List differencerecords = [] def DifferenceRecord dr = new DifferenceRecord() dr.productname = meta.productinfo.productname dr.subcomponentname = meta.productinfo.subcomponentname dr.rawdifferencetext = line // Example of raw text to process: // 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py (sha, line2) = extractsha(line) dr.sha = sha (result, squashinfo) = extractsquashes(line2) if (result) { dr.type = "Squash" dr.squashinfo = squashinfo squashrecords = fillsquashrecords(sourcepath, sha) differencerecords.addAll(squashrecords) } else { (result, mergeinfo) = extractmerges(line2) if (result) { dr.type = "Merge" dr.mergeinfo = mergeinfo } else { (result, jiraticketinfo, comment) = extractreverts(line2) if (result) { dr.type = "Revert" dr.jiraticketinfo = jiraticketinfo dr.comment = comment } else { (result, jiraticketinfo, comment) = extractjiraticketinfo(line2) if (result) { dr.type = "Ticketed" dr.jiraticketinfo = jiraticketinfo dr.comment = comment } else { dr.type = "Unticketed" dr.comment = line2 } } } } differencerecords.add(dr) return differencerecords } def List processtext(meta, String text, String sourcepath) { def List differencerecords = [] text.split('\n').each { def line = it if (line?.trim()) { def List drs = fillrecord(meta, line.trim(), sourcepath) differencerecords.addAll(drs) //echo "${dr.dump()}" } } return differencerecords } def getdifferencerecords(earlymeta, latermeta, String sourcepath) { boolean result = true; def List differencerecords = [] def earlygitinfo = earlymeta.gitinfo def latergitinfo = latermeta.gitinfo if (earlygitinfo.remoteurl == latergitinfo.remoteurl) { def earlyrevision = earlygitinfo.gitrevision def laterrevision = latergitinfo.gitrevision echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}" summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}") def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision) differencerecords = processtext(earlymeta, differencetext, sourcepath) if (earlyrevision == laterrevision) { summaryappend(" No change") } } else { summaryappend("The early and later git urls don't match") echo "The early and later git urls don't match" result = false; setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}") } return [result, differencerecords] } def deletedcomponent(earlymetaitems, key) { echo "Component ${key} has been deleted in later version" summaryappend("Deleted ") } def addedcomponent(latermetaitems, key) { echo "Component ${key} has been added in later version" summaryappend("Added ") } class SquashReportRecord { String comment int duplicates SquashReportRecord() { comment = '' duplicates = 0 } } def squashreport(String reportsubfolder, String title, LinkedHashMap squashes, boolean iscomponent = true) { dir(reportsubfolder) { def String squashlist = "${title}\nSquash list:\n" def squashessorted = squashes.sort() for (asquash in squashessorted) { def SquashReportRecord s = asquash.value squashlist += "${s.comment} (${s.duplicates})\n" } writeFile(file: "squashlist.txt", text: squashlist) } } class MergeReportRecord { String comment int duplicates MergeReportRecord() { comment = '' duplicates = 0 } } def mergereport(String reportsubfolder, String title, LinkedHashMap merges, boolean iscomponent = true) { dir(reportsubfolder) { def String mergelist = "${title}\nMerge list:\n" def mergessorted = merges.sort() for (amerge in mergessorted) { def MergeReportRecord m = amerge.value mergelist += "${m.comment} (${m.duplicates})\n" } writeFile(file: "mergelist.txt", text: mergelist) } } class CommentsRecord { String comment int duplicates CommentsRecord() { comment = '' duplicates = 0 } } class RevertsReportRecord { String jiraticket String ticketURI LinkedHashMap commentscollection RevertsReportRecord() { jiraticket = '' ticketURI = '' commentscollection = new LinkedHashMap() } } def revertsreport(String reportsubfolder, String title, LinkedHashMap reverts, boolean iscomponent = true) { dir(reportsubfolder) { def String revertlist = "${title}\nRevert list:\n" def revertssorted = reverts.sort() for (arevert in revertssorted) { def RevertsReportRecord r = arevert.value revertlist += "${r.jiraticket} (${r.ticketURI}):\n" def commentscollection = r.commentscollection def sortedcommentscollection = commentscollection.sort() for (commentrecord in sortedcommentscollection) { def CommentsRecord c = commentrecord.value revertlist += "${c.comment} (${c.duplicates})\n" } revertlist += "\n" } writeFile(file: "revertlist.txt", text: revertlist) } } class TicketedReportRecord { String jiraticket String ticketURI LinkedHashMap commentscollection TicketedReportRecord() { jiraticket = '' ticketURI = '' commentscollection = new LinkedHashMap() } } def ticketedreport(String reportsubfolder, String title, LinkedHashMap ticketed, boolean iscomponent = true) { dir(reportsubfolder) { def String ticketedlist = "${title}\nTicketed list:\n" def ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { def TicketedReportRecord r = aticketed.value ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n" def commentscollection = r.commentscollection def sortedcommentscollection = commentscollection.sort() for (commentrecord in sortedcommentscollection) { def CommentsRecord c = commentrecord.value ticketedlist += "${c.comment} (${c.duplicates})\n" } ticketedlist += "\n" } writeFile(file: "ticketedlist.txt", text: ticketedlist) } } class UnticketedReportRecord { String comment int duplicates UnticketedReportRecord() { comment = '' duplicates = 0 } } def unticketedreport(String reportsubfolder, String title, LinkedHashMap unticketed, boolean iscomponent = true) { dir(reportsubfolder) { def String unticketedlist = "${title}\nUnticketed list:\n" def unticketedsorted = unticketed.sort() for (anunticketed in unticketedsorted) { UnticketedReportRecord u = anunticketed.value unticketedlist += "${u.comment} (${u.duplicates})\n" } writeFile(file: "unticketedlist.txt", text: unticketedlist) } } class TicketsReportRecord { String jiraticket String ticketURI int duplicates TicketsReportRecord() { jiraticket = '' ticketURI = '' duplicates = 0 } } def ticketsreport(String reportsubfolder, String title, tickets, boolean iscomponent = true) { dir(reportsubfolder) { String ticketlist = "${title}\nTicket list:\n" ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { TicketsReportRecord t = ticketnumber.value ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n" } writeFile(file: "ticketlist.txt", text: ticketlist) } } def differencerecordsreport(String reportsubfolder, String title, differencerecords, boolean iscomponent = true) { dir(reportsubfolder) { String recordlist = "${title}\n" for (dr in differencerecords) { recordlist += dr.dump() + "\n\n" } writeFile(file: "recordlist.txt", text: recordlist) } } def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] def Map squashes = [:] def Map merges = [:] def Map reverts = [:] def Map unticketed = [:] def Map tickets = [:] for (dr in differencerecords) { if (dr.type == "Squash") { def SquashReportRecord s if (!squashes.containsKey(dr.mergeinfo)) { s = new SquashReportRecord() s.comment = dr.squashinfo s.duplicates = 1 } else { s = squashes[dr.squashinfo] s.duplicates += 1 } squashes[dr.squashinfo] = s } if (dr.type == "Merge") { def MergeReportRecord m if (!merges.containsKey(dr.mergeinfo)) { m = new MergeReportRecord() m.comment = dr.mergeinfo m.duplicates = 1 } else { m = merges[dr.mergeinfo] m.duplicates += 1 } merges[dr.mergeinfo] = m } if (dr.type == "Revert") { def RevertsReportRecord r for (ticket in dr.jiraticketinfo) { JiraTicketInfo j = ticket if (!reverts.containsKey(j.jiranumber)) { r = new RevertsReportRecord() r.jiraticket = j.jiranumber r.ticketURI = j.uri CommentsRecord c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 r.commentscollection[dr.comment] = c } else { r = reverts[j.jiranumber] def CommentsRecord c if (!r.commentscollection.containsKey(dr.comment)) { c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 r.commentscollection[dr.comment] = c } else { c = r.commentscollection[dr.comment] c.duplicates += 1 } r.commentscollection[dr.comment] = c } reverts[j.jiranumber] = r def TicketsReportRecord t if (!tickets.containsKey(j.jiranumber)) { t = new TicketsReportRecord() t.jiraticket = j.jiranumber t.ticketURI = j.uri t.duplicates = 1 } else { t = tickets[j.jiranumber] t.duplicates += 1 } tickets[j.jiranumber] = t } } if (dr.type == "Ticketed") { def TicketedReportRecord t for (ticket in dr.jiraticketinfo) { JiraTicketInfo j = ticket if (!ticketed.containsKey(j.jiranumber)) { t = new TicketedReportRecord() t.jiraticket = j.jiranumber t.ticketURI = j.uri CommentsRecord c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 t.commentscollection[dr.comment] = c } else { t = ticketed[j.jiranumber] def CommentsRecord c if (!t.commentscollection.containsKey(dr.comment)) { c = new CommentsRecord() c.comment = dr.comment c.duplicates = 1 t.commentscollection[dr.comment] = c } else { c = t.commentscollection[dr.comment] c.duplicates += 1 } t.commentscollection[dr.comment] = c } ticketed[j.jiranumber] = t def TicketsReportRecord ti if (!tickets.containsKey(j.jiranumber)) { ti = new TicketsReportRecord() ti.jiraticket = j.jiranumber ti.ticketURI = j.uri ti.duplicates = 1 } else { ti = tickets[j.jiranumber] ti.duplicates += 1 } tickets[j.jiranumber] = ti } } if (dr.type == "Unticketed") { UnticketedReportRecord u if (!unticketed.containsKey(dr.comment)) { u = new UnticketedReportRecord() u.comment = dr.comment u.duplicates = 1 } else { u = unticketed[dr.comment] u.duplicates += 1 } unticketed[dr.comment] = u } } String reportsubfolder = "" if (iscomponent) { reportsubfolder = "${reportfolder}/${meta.componentidentifier()}" } else { reportsubfolder = "${reportfolder}/Combined" } mergereport(reportsubfolder, title, merges, iscomponent) squashreport(reportsubfolder, title, squashes, iscomponent) revertsreport(reportsubfolder, title, reverts, iscomponent) ticketedreport(reportsubfolder, title, ticketed, iscomponent) unticketedreport(reportsubfolder, title, unticketed, iscomponent) ticketsreport(reportsubfolder, title, tickets, iscomponent) differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) } def title(earlymeta, latermeta) { return """ Differences in: Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} between: then: Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} now: Build date: ${latermeta.jenkinsbuildinfo.builddate} Build stamp: ${latermeta.buildstampinfo.buildstamp} Git revision: ${latermeta.gitinfo.gitrevision} """ } def compare(earlymetaitems, latermetaitems, key) { boolean result = true; echo "Collecting changes between early and later versions of component: ${key}" def earlymeta = earlymetaitems[key].meta def latermeta = latermetaitems[key].meta summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ") def boolean repoavailable def String uniquefolderforsourcerepo (repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta) List differencerecords = [] if (repoavailable) { (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) summaryappend(" Items: ${differencerecords.size()} ") report(earlymeta, title(earlymeta, latermeta), differencerecords) } else { summaryappend(" No source repo ") } summaryappend("\n") return [result, differencerecords]; } def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) { def List differencerecords = [] def int md = maxdepth.toInteger() resetfolders() preconditions() echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}" def earlymeta = meta.importrelated(true, earlymetafolder) def latermeta = meta.importrelated(true, latermetafolder) def earlymetaitems = earlymeta.releasewalkerunique(md) def latermetaitems = latermeta.releasewalkerunique(md) echo "Early items:" showmetaitems(earlymetaitems) echo "Later items:" showmetaitems(latermetaitems) for (key in earlymetaitems.keySet()) { if (latermetaitems.containsKey(key)) { (result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key) differencerecords = differencerecords + componentdifferencerecords } else { summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ") deletedcomponent(earlymetaitems, key) summaryappend("\n") } } for (key in latermetaitems.keySet()) { if (!earlymetaitems.containsKey(key)) { summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ") addedcomponent(latermetaitems, key) summaryappend("\n") } } report(meta, title(earlymeta, latermeta), differencerecords, false) return [summarystring, markunstable, unstablereason] } return this;