Merge commit 'e9ce58fc62' into 1.10005.jELGIP74_Common_scripts

This commit is contained in:
Dan Hawkins 2019-07-23 13:33:03 +01:00
commit 93e10fe6da
12 changed files with 1269 additions and 85 deletions

View file

@ -9,7 +9,7 @@ def registerPipelineScriptConfiguration(meta) {
}
def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) {
gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone)
gh.gitCheckoutAlt(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone)
registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir))
}

View file

@ -14,6 +14,7 @@ pipeline {
cleanfolders = [
"${WORKSPACE}/logic",
"${WORKSPACE}/embedded_logic",
"${WORKSPACE}/diff",
]
cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy")
cleaner.clean(cleanfolders)
@ -36,7 +37,7 @@ pipeline {
script {
def artifactlist = meta.generateartifacts(["embedded_logic/**"])
sh """
rm -v -rf meta
rm -v -rf diff meta embedded_logic
mv -v logic/* .
"""
archiveArtifacts "${artifactlist}"

View file

@ -0,0 +1,744 @@
import groovy.json.JsonSlurper
markunstable = false
unstablereason = ""
summaryfolder = "${WORKSPACE}/summary"
summaryfile = "${summaryfolder}/summary.txt"
summarystring = ""
sourcerepofolder = "${WORKSPACE}/sourcerepos"
tempfolder = "${WORKSPACE}/temp"
reportfolder = "${WORKSPACE}/reports"
class JiraTicketInfo {
String jiranumber
String uri
JiraTicketInfo() {
jiranumber = ''
uri = ''
}
def String dump() {
return "jiranumber: ${jiranumber} uri: ${uri}"
}
}
class DifferenceRecord {
String productname
String subcomponentname
String jirakey
String rawdifferencetext
String sha
String type
String mergeinfo
List<JiraTicketInfo> jiraticketinfo
String comment
DifferenceRecord() {
productname = ""
subcomponentname = ""
jirakey = ""
rawdifferencetext = ""
sha = ""
type = ""
mergeinfo = ""
jiraticketinfo = []
comment = ""
}
def String dump() {
String jiraticketstring = ''
for (j in jiraticketinfo) {
jiraticketstring += j.dump() + " "
}
return "Difference Record:\nproductname: ${productname}\nsubcomponentname: ${subcomponentname}\njirakey: ${jirakey}\nrawdifferencetext: ${rawdifferencetext}\nsha: ${sha}\ntype: ${type}\nmergeinfo: ${mergeinfo}\njiraticketinfo: ${jiraticketstring}\ncomment: ${comment}"
}
}
def resetfolders() {
dir(summaryfolder) {
deleteDir()
}
dir(tempfolder) {
deleteDir()
}
dir(reportfolder) {
deleteDir()
}
}
def preconditions() {
markunstable = false
unstablereason = ""
summarystring = ""
writeFile(file: summaryfile, text: "Release summary: \n")
}
def setunstable(String reason) {
markunstable = true
unstablereason += reason + "\n"
echo "Marked build as unstable: Reason: ${reason}"
}
def summaryappend(String summary) {
def String currentsummarycontents = readFile(file: summaryfile)
writeFile(file: summaryfile, text: currentsummarycontents + summary)
summarystring += summary
}
def showmetaitems(metaitems) {
String info = ''
for (key in metaitems.keySet()) {
def relatedmetainfo = metaitems[key]
def offsetpath = relatedmetainfo.offsetpath
info += "Name: ${key}\t\t\t\tPath: ${offsetpath}\n"
}
echo info
}
def boolean checkoutsourcerepo(buildmeta, String checkoutfolder) {
boolean checkedout = false
def String sourcerepo = buildmeta.gitinfo.remoteurl
if (sourcerepo?.trim()) {
echo "Updating source repo: ${sourcerepo} in folder: ${checkoutfolder}"
if (sourcerepo) {
// Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component
if (fileExists(checkoutfolder)) {
dir(checkoutfolder) {
sh("""
git reset --hard
git clean -fdx
git pull
"""
)
}
}
else {
dir(sourcerepofolder) {
sh("git clone ${sourcerepo} ${checkoutfolder}")
}
}
}
checkedout = true
}
return checkedout
}
def String getcomponent(meta) {
def String uniquefolder = "${meta.productinfo.productname}_${meta.productinfo.subcomponentname}"
def String uniquefoldertemp = "${tempfolder}/${uniquefolder}"
def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}"
boolean checkedout = checkoutsourcerepo(meta, uniquefolderforsourcerepo)
return [checkedout, uniquefolderforsourcerepo]
}
def getgitlogdifferences(String gitrootfolder, String earlyrevision, String laterrevision, boolean tripledot = false) {
def differencerecords = []
def String releaseexpression
if (tripledot) {
releaseexpression = earlyrevision + "..." + laterrevision
} else {
releaseexpression = earlyrevision + ".." + laterrevision
}
def String differencetext
dir(gitrootfolder) {
differencetext = sh(returnStdout: true, script: "git --no-pager log --oneline ${releaseexpression}").trim()
}
return differencetext
}
def extractsha(String line) {
def pattern = ~"^(\\p{XDigit}*) (.*)"
def match = (line =~ pattern)
match.find()
def sha = match.group(1)
def restofline = match.group(2)
//echo "Sha filtering: line: ${line} sha: ${sha} therest: ${restofline}"
return [sha.trim(), restofline.trim()]
}
def extractmerges(String line) {
def pattern1 = "^(.*)(Merge branch .*)"
def match1 = (line =~ pattern1)
if (match1.find()) {
return [true, match1.group(2)]
}
def pattern2 = "^(.*)(Merge remote-tracking branch .*)"
def match2 = (line =~ pattern2)
if (match2.find()) {
return [true, match2.group(2)]
}
def pattern3 = "^(.*)(Merge commit .*)"
def match3 = (line =~ pattern3)
if (match3.find()) {
return [true, match3.group(2)]
}
def pattern4 = "^(.*)(Add .* from commit .*)"
def match4 = (line =~ pattern4)
if (match4.find()) {
return [true, match4.group(2)]
}
return [false, line]
}
def jiraticketexists(String jiraticket) {
def String URI = "${JIRA_URI}/browse/${jiraticket}"
def String RestfulURI = "${JIRA_URI}/rest/api/latest/issue/${jiraticket}?fields=summary"
def get = new URL(RestfulURI).openConnection();
get.setRequestProperty( 'X-AUSERNAME', "${JIRA_USERNAME}" )
get.setRequestProperty( 'Authorization', "Basic ${JIRA_ENCODED_PASSWORD}" )
get.setRequestProperty( 'User-Agent', 'groovy-2.4.4' )
get.setRequestProperty( 'Accept', 'application/json' )
def getRC = get.getResponseCode();
def String redirectedjiraticket = jiraticket
if(!getRC.equals(200)) {
return [false, '', jiraticket]
} else {
def jsonresponse = new JsonSlurper().parseText(get.inputStream.getText('UTF-8'))
redirectedjiraticket = jsonresponse['key']
}
return [true, URI, redirectedjiraticket]
}
def extractjiraticketinfo(String line) {
def List jiratickets = []
boolean exists = false
String URI = ''
while (true) {
line = line.trim()
def commapattern = '^\\s*,(.*)' // Consume separating commas at the beginning of the line
def commamatch = (line =~ commapattern)
if (commamatch.find()) {
line = commamatch.group(1).trim()
}
def pattern1 = '^(\\w+-\\d+)(.*)' //GIS-1234 hmbxr5601-1234
def match1 = (line =~ pattern1)
if (match1.find()) {
jiraticket = match1.group(1)
(exists, URI, jiraticket) = jiraticketexists(jiraticket)
if (exists) {
def JiraTicketInfo j = new JiraTicketInfo()
j.jiranumber = jiraticket
j.uri = URI
jiratickets.add(j)
line = match1.group(2)
} else {
break
}
} else {
def pattern2 = '^(\\w+)(\\d+)(.*)' //GIS1234 smb1234
def match2 = (line =~ pattern2)
if (match2.find()) {
jiraticket = match2.group(1) + "-" + match2.group(2)
(exists, URI, jiraticket) = jiraticketexists(jiraticket)
if (exists) {
def JiraTicketInfo j = new JiraTicketInfo()
j.jiranumber = jiraticket
j.uri = URI
jiratickets.add(j)
line = match2.group(3)
} else {
break
}
} else {
def pattern3 = '^(\\w+)\\s(\\d+)(.*)' //GIS 1234 smb 1234
def match3 = (line =~ pattern3)
if (match3.find()) {
jiraticket = match3.group(1) + "-" + match3.group(2)
(exists, URI, jiraticket) = jiraticketexists(jiraticket)
if (exists) {
def JiraTicketInfo j = new JiraTicketInfo()
j.jiranumber = jiraticket
j.uri = URI
jiratickets.add(j)
line = match3.group(3)
} else {
break
}
} else {
break
}
}
}
}
boolean result = jiratickets.size() > 0
return [result, jiratickets, line]
}
def extractreverts(String line) {
boolean result = true
def pattern1 = '^(.*)Revert \"(.*)\"'
def match1 = (line =~ pattern1)
if (match1.find()) {
def line2 = match1.group(2)
(result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2)
return [true, originaljiraticketinfo, originalcomment]
}
def pattern2 = '^(.*)Partial Revert \"(.*)\"'
def match2 = (line =~ pattern2)
if (match2.find()) {
def line2 = match2.group(2)
(result, originaljiraticketinfo, originalcomment) = extractjiraticketinfo(line2)
return [true, originaljiraticketinfo, originalcomment]
}
return [false, [], line]
}
def DifferenceRecord fillrecord(meta, line) {
def DifferenceRecord dr = new DifferenceRecord()
dr.productname = meta.productinfo.productname
dr.subcomponentname = meta.productinfo.subcomponentname
dr.jirakey = meta.productinfo.jirakey
dr.rawdifferencetext = line
// Example of raw text to process:
// 76056562e4 GIS-1621 - Removing redundant lines and effects of this - Restore nuget_version_edit.py
(sha, line2) = extractsha(line)
dr.sha = sha
(result, mergeinfo) = extractmerges(line2)
if (result) {
dr.type = "Merge"
dr.mergeinfo = mergeinfo
} else {
(result, jiraticketinfo, comment) = extractreverts(line2)
if (result) {
dr.type = "Revert"
dr.jiraticketinfo = jiraticketinfo
dr.comment = comment
} else {
(result, jiraticketinfo, comment) = extractjiraticketinfo(line2)
if (result) {
dr.type = "Ticketed"
dr.jiraticketinfo = jiraticketinfo
dr.comment = comment
} else {
dr.type = "Unticketed"
dr.comment = line2
}
}
}
return dr
}
def List processtext(meta, String text) {
def List differencerecords = []
text.split('\n').each {
def line = it
if (line?.trim()) {
def DifferenceRecord dr = fillrecord(meta, line.trim())
differencerecords.add(dr)
//echo "${dr.dump()}"
}
}
return differencerecords
}
def getdifferencerecords(earlymeta, latermeta, String sourcepath) {
boolean result = true;
def List differencerecords = []
def earlygitinfo = earlymeta.gitinfo
def latergitinfo = latermeta.gitinfo
if (earlygitinfo.remoteurl == latergitinfo.remoteurl) {
def earlyrevision = earlygitinfo.gitrevision
def laterrevision = latergitinfo.gitrevision
echo "Comparing early revision: ${earlyrevision} with: ${laterrevision} Working copy path: ${sourcepath}"
summaryappend("Early revision: ${earlyrevision} Later revision: ${laterrevision}")
def String differencetext = getgitlogdifferences(sourcepath, earlyrevision, laterrevision)
differencerecords = processtext(earlymeta, differencetext)
if (earlyrevision == laterrevision) {
summaryappend(" No change")
}
} else {
summaryappend("The early and later git urls don't match")
echo "The early and later git urls don't match"
result = false;
setunstable("The early and later git urls don't match in: ${earlymeta.componentidentifier()}")
}
return [result, differencerecords]
}
def deletedcomponent(earlymetaitems, key) {
echo "Component ${key} has been deleted in later version"
summaryappend("Deleted ")
}
def addedcomponent(latermetaitems, key) {
echo "Component ${key} has been added in later version"
summaryappend("Added ")
}
class MergeReportRecord {
String comment
int duplicates
MergeReportRecord() {
comment = ''
duplicates = 0
}
}
def mergereport(String reportsubfolder, LinkedHashMap<String, MergeReportRecord> merges, boolean iscomponent = true) {
dir(reportsubfolder) {
def String mergelist = "Merge list:\n"
def mergessorted = merges.sort()
for (amerge in mergessorted) {
def MergeReportRecord m = amerge.value
mergelist += "${m.comment} (${m.duplicates})\n"
}
writeFile(file: "mergelist.txt", text: mergelist)
}
}
class CommentsRecord {
String comment
int duplicates
CommentsRecord() {
comment = ''
duplicates = 0
}
}
class RevertsReportRecord {
String jiraticket
String ticketURI
LinkedHashMap<String, CommentsRecord> commentscollection
RevertsReportRecord() {
jiraticket = ''
ticketURI = ''
commentscollection = new LinkedHashMap<String, CommentsRecord>()
}
}
def revertsreport(String reportsubfolder, LinkedHashMap<String, RevertsReportRecord> reverts, boolean iscomponent = true) {
dir(reportsubfolder) {
def String revertlist = "Revert list:\n"
def revertssorted = reverts.sort()
for (arevert in revertssorted) {
def RevertsReportRecord r = arevert.value
revertlist += "${r.jiraticket} (${r.ticketURI}):\n"
def commentscollection = r.commentscollection
def sortedcommentscollection = commentscollection.sort()
for (commentrecord in sortedcommentscollection) {
def CommentsRecord c = commentrecord.value
revertlist += "${c.comment} (${c.duplicates})\n"
}
revertlist += "\n"
}
writeFile(file: "revertlist.txt", text: revertlist)
}
}
class TicketedReportRecord {
String jiraticket
String ticketURI
LinkedHashMap<String, CommentsRecord> commentscollection
TicketedReportRecord() {
jiraticket = ''
ticketURI = ''
commentscollection = new LinkedHashMap<String, CommentsRecord>()
}
}
def ticketedreport(String reportsubfolder, LinkedHashMap<String, TicketedReportRecord> ticketed, boolean iscomponent = true) {
dir(reportsubfolder) {
def String ticketedlist = "Ticketed list:\n"
def ticketedsorted = ticketed.sort()
for (aticketed in ticketedsorted) {
def TicketedReportRecord r = aticketed.value
ticketedlist += "${r.jiraticket} (${r.ticketURI}):\n"
def commentscollection = r.commentscollection
def sortedcommentscollection = commentscollection.sort()
for (commentrecord in sortedcommentscollection) {
def CommentsRecord c = commentrecord.value
ticketedlist += "${c.comment} (${c.duplicates})\n"
}
ticketedlist += "\n"
}
writeFile(file: "ticketedlist.txt", text: ticketedlist)
}
}
class UnticketedReportRecord {
String comment
int duplicates
UnticketedReportRecord() {
comment = ''
duplicates = 0
}
}
def unticketedreport(String reportsubfolder, LinkedHashMap<String, UnticketedReportRecord> unticketed, boolean iscomponent = true) {
dir(reportsubfolder) {
def String unticketedlist = "Unticketed list:\n"
def unticketedsorted = unticketed.sort()
for (anunticketed in unticketedsorted) {
UnticketedReportRecord u = anunticketed.value
unticketedlist += "${u.comment} (${u.duplicates})\n"
}
writeFile(file: "unticketedlist.txt", text: unticketedlist)
}
}
class TicketsReportRecord {
String jiraticket
String ticketURI
int duplicates
TicketsReportRecord() {
jiraticket = ''
ticketURI = ''
duplicates = 0
}
}
def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) {
dir(reportsubfolder) {
String ticketlist = "Ticket list:\n"
ticketssorted = tickets.sort()
for (ticketnumber in ticketssorted) {
TicketsReportRecord t = ticketnumber.value
ticketlist += "${t.jiraticket} (${t.ticketURI}) (${t.duplicates})\n"
}
writeFile(file: "ticketlist.txt", text: ticketlist)
}
}
def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) {
dir(reportsubfolder) {
String recordlist = ""
for (dr in differencerecords) {
recordlist += dr.dump() + "\n\n"
}
writeFile(file: "recordlist.txt", text: recordlist)
}
}
def boolean report(meta, differencerecords, boolean iscomponent = true) {
def Map ticketed = [:]
def Map merges = [:]
def Map reverts = [:]
def Map unticketed = [:]
def Map tickets = [:]
for (dr in differencerecords) {
if (dr.type == "Merge") {
def MergeReportRecord m
if (!merges.containsKey(dr.mergeinfo)) {
m = new MergeReportRecord()
m.comment = dr.mergeinfo
m.duplicates = 1
} else {
m = merges[dr.mergeinfo]
m.duplicates += 1
}
merges[dr.mergeinfo] = m
}
if (dr.type == "Revert") {
def RevertsReportRecord r
for (ticket in dr.jiraticketinfo) {
JiraTicketInfo j = ticket
if (!reverts.containsKey(j.jiranumber)) {
r = new RevertsReportRecord()
r.jiraticket = j.jiranumber
r.ticketURI = j.uri
CommentsRecord c = new CommentsRecord()
c.comment = dr.comment
c.duplicates = 1
r.commentscollection[dr.comment] = c
} else {
r = reverts[j.jiranumber]
def CommentsRecord c
if (!r.commentscollection.containsKey(dr.comment)) {
c = new CommentsRecord()
c.comment = dr.comment
c.duplicates = 1
r.commentscollection[dr.comment] = c
} else {
c = r.commentscollection[dr.comment]
c.duplicates += 1
}
r.commentscollection[dr.comment] = c
}
reverts[j.jiranumber] = r
def TicketsReportRecord t
if (!tickets.containsKey(j.jiranumber)) {
t = new TicketsReportRecord()
t.jiraticket = j.jiranumber
t.ticketURI = j.uri
t.duplicates = 1
} else {
t = tickets[j.jiranumber]
t.duplicates += 1
}
tickets[j.jiranumber] = t
}
}
if (dr.type == "Ticketed") {
def TicketedReportRecord t
for (ticket in dr.jiraticketinfo) {
JiraTicketInfo j = ticket
if (!ticketed.containsKey(j.jiranumber)) {
t = new TicketedReportRecord()
t.jiraticket = j.jiranumber
t.ticketURI = j.uri
CommentsRecord c = new CommentsRecord()
c.comment = dr.comment
c.duplicates = 1
t.commentscollection[dr.comment] = c
} else {
t = ticketed[j.jiranumber]
def CommentsRecord c
if (!t.commentscollection.containsKey(dr.comment)) {
c = new CommentsRecord()
c.comment = dr.comment
c.duplicates = 1
t.commentscollection[dr.comment] = c
} else {
c = t.commentscollection[dr.comment]
c.duplicates += 1
}
t.commentscollection[dr.comment] = c
}
ticketed[j.jiranumber] = t
def TicketsReportRecord ti
if (!tickets.containsKey(j.jiranumber)) {
ti = new TicketsReportRecord()
ti.jiraticket = j.jiranumber
ti.ticketURI = j.uri
ti.duplicates = 1
} else {
ti = tickets[j.jiranumber]
ti.duplicates += 1
}
tickets[j.jiranumber] = ti
}
}
if (dr.type == "Unticketed") {
UnticketedReportRecord u
if (!unticketed.containsKey(dr.comment)) {
u = new UnticketedReportRecord()
u.comment = dr.comment
u.duplicates = 1
} else {
u = unticketed[dr.comment]
u.duplicates += 1
}
unticketed[dr.comment] = u
}
}
String reportsubfolder = ""
if (iscomponent) {
reportsubfolder = "${reportfolder}/${meta.componentidentifier()}"
} else {
reportsubfolder = "${reportfolder}/Combined"
}
mergereport(reportsubfolder, merges, iscomponent)
revertsreport(reportsubfolder, reverts, iscomponent)
ticketedreport(reportsubfolder, ticketed, iscomponent)
unticketedreport(reportsubfolder, unticketed, iscomponent)
ticketsreport(reportsubfolder, tickets, iscomponent)
differencerecordsreport(reportsubfolder, differencerecords, iscomponent)
}
def compare(earlymetaitems, latermetaitems, key) {
boolean result = true;
echo "Collecting changes between early and later versions of component: ${key}"
def earlymeta = earlymetaitems[key].meta
def latermeta = latermetaitems[key].meta
summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: Changed: ")
def boolean repoavailable
def String uniquefolderforsourcerepo
(repoavailable, uniquefolderforsourcerepo) = getcomponent(earlymeta)
List differencerecords = []
if (repoavailable) {
(result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo)
summaryappend(" Items: ${differencerecords.size()} ")
report(earlymeta, differencerecords)
} else {
summaryappend(" No source repo ")
}
summaryappend("\n")
return [result, differencerecords];
}
def comparehierarchy(meta, String earlymetafolder, String latermetafolder, String maxdepth, String summaryfolder) {
def List differencerecords = []
def int md = maxdepth.toInteger()
resetfolders()
preconditions()
echo "Comparing product meta hierarchy from folders: ${earlymetafolder} and ${latermetafolder} to a depth of ${md}. Exporting summary to ${summaryfolder}"
def earlymeta = meta.importrelated(earlymetafolder)
def latermeta = meta.importrelated(latermetafolder)
def earlymetaitems = earlymeta.releasewalkerunique(md)
def latermetaitems = latermeta.releasewalkerunique(md)
echo "Early items:"
showmetaitems(earlymetaitems)
echo "Later items:"
showmetaitems(latermetaitems)
for (key in earlymetaitems.keySet()) {
if (latermetaitems.containsKey(key)) {
(result, componentdifferencerecords) = compare(earlymetaitems, latermetaitems, key)
differencerecords = differencerecords + componentdifferencerecords
} else {
summaryappend("${earlymeta.productinfo.productname} ${earlymeta.productinfo.subcomponentname}: ")
deletedcomponent(earlymetaitems, key)
summaryappend("\n")
}
}
for (key in latermetaitems.keySet()) {
if (!earlymetaitems.containsKey(key)) {
summaryappend("${latermeta.productinfo.productname} ${latermeta.productinfo.subcomponentname}: ")
addedcomponent(latermetaitems, key)
summaryappend("\n")
}
}
report(meta, differencerecords, false)
return [summarystring, markunstable, unstablereason]
}
return this;

View file

@ -0,0 +1,91 @@
pipeline {
agent { label "${AGENT_LABEL}" }
stages {
stage('Initial setup') {
steps {
script {
autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy")
meta = autobuilderscript.startnonbuild(this)
}
}
}
stage('Clean exports') {
steps {
script {
SUMMARYDIR = "${WORKSPACE}/summary"
REPORTDIR = "${WORKSPACE}/reports"
EARLYROOTDIR = 'early'
EARLYDIR = "${WORKSPACE}/${EARLYROOTDIR}"
LATERROOTDIR = 'later'
LATERDIR = "${WORKSPACE}/${LATERROOTDIR}"
cleanfolders = [
"${WORKSPACE}/build_artifacts",
"${EARLYDIR}",
"${LATERDIR}",
"${SUMMARYDIR}",
"${REPORTDIR}",
]
cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy")
cleaner.clean(cleanfolders)
}
}
}
stage('Obtain top level meta') {
steps {
script {
importer = load("${autobuilder.commonscripts()}/import/importer.groovy")
SOURCEURL = TOP_LEVEL_META_REPOSITORY.trim()
importer.artifactimport(meta, EARLY_SOURCE, EARLY_BUILD_JOB, EARLY_BUILD_SELECTOR, SOURCEURL, RELEASE_EARLY_REVISION, EARLYROOTDIR, false, 'meta/**')
importer.artifactimport(meta, LATER_SOURCE, LATER_BUILD_JOB, LATER_BUILD_SELECTOR, SOURCEURL, RELEASE_LATER_REVISION, LATERROOTDIR, false, 'meta/**')
}
}
}
stage('Diff components') {
steps {
script {
def comparator = load("${autobuilder.commonscripts()}/differences/dometahierarchy.groovy")
(summarystring, markunstable, unstablereason) = comparator.comparehierarchy(meta, EARLYDIR, LATERDIR, MAX_META_DEPTH, SUMMARYDIR)
if (markunstable) {
env.unstablereason = unstablereason
currentBuild.result = 'UNSTABLE'
}
}
}
}
stage('Archiving artifacts') {
steps {
script {
def artifactlist = meta.generateartifacts()
archiveArtifacts "summary/**,reports/**,${artifactlist}"
}
}
}
stage('Summarise') {
steps {
script {
echo "Summary:\n${summarystring}\n"
if (env.unstablereason) {
echo "Instability reason: ${env.unstablereason}"
}
}
}
}
}
post {
always {
echo 'Post-build'
}
failure {
echo 'Failed build'
}
unstable {
echo 'Unstable build'
}
success {
echo 'Succeeded build'
}
}
}

View file

@ -17,7 +17,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, bo
return importedmeta
}
def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) {
def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) {
copyArtifacts(
filter: filter,
excludes: excludes,
@ -42,11 +42,14 @@ def rewritemetafileasreleased(meta, String filepath) {
meta.saveJsonSerialised(filepath)
}
def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) {
publisher = load("${autobuilder.commonscripts()}/publish/do.groovy")
def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) {
def rooturl = repo.trim()
def rootrevision = revision.trim()
echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}"
gh.gitCheckout(rooturl, rootrevision, destpath)
def ArrayList<String> sparseitemlist = gh.convertSparseItemsString(filter)
gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist)
def importedmeta = meta.importrelated(destpath)
@ -59,8 +62,11 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath,
def clonerevision = "refs/tags/${relatedmeta.releasetag()}"
def clonepath = "${destpath}/${relatedoffsetpath}"
echo "Cloning release repo: ${cloneurl}, revision: ${clonerevision} at folder: ${clonepath}"
gh.gitCheckout(cloneurl, clonerevision, clonepath)
gh.gitCheckoutAlt(cloneurl, clonerevision, clonepath, "", 1, true, [])
gh.stripgithistory(clonepath)
def metafilepath = "${clonepath}/${relatedmeta.metafilepath()}"
echo "Marking related meta as released and rewriting meta file: ${metafilepath}"
rewritemetafileasreleased(relatedmeta, metafilepath)
@ -68,7 +74,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath,
gh.stripgithistory(destpath)
def metafilepath = "${destpath}/${meta.metafilepath()}"
echo "Marking root meta as released and rewriting meta file: ${metafilepath}"
rewritemetafileasreleased(meta, metafilepath)
rewritemetafileasreleased(importedmeta, metafilepath)
if (incrementversionnumber) {
meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))
@ -76,11 +82,35 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath,
return importedmeta
}
def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) {
def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) {
def rooturl = repo.trim()
def rootrevision = revision.trim()
echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}"
def ArrayList<String> sparseitemlist = gh.convertSparseItemsString(filter)
gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist)
def importedmeta = meta.importrelated(destpath)
gh.stripgithistory(destpath)
def metafilepath = "${destpath}/${meta.metafilepath()}"
echo "Marking root meta as released and rewriting meta file: ${metafilepath}"
rewritemetafileasreleased(importedmeta, metafilepath)
if (incrementversionnumber) {
meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision))
}
return importedmeta
}
def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) {
switch (source) {
case "Jenkins job artifact":
return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint)
case "Release repository":
if (nested) {
return hierarchyartifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint)
}
return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint)
default:
error("Unknown type of artifact source: ${source}")

View file

@ -1,3 +1,27 @@
import hudson.FilePath
// Transfer file from intermediate position to final position;
// creating necessary folders on the way;
// deleting intermediate file
def copyToFinalPosition(String pathToIntermediate, String finalBaseFolder, String finalFilename) {
def channel = Jenkins.getInstance().getComputer(env['NODE_NAME']).getChannel()
def String abs_exports_root_path = "${WORKSPACE}/${finalBaseFolder}"
def String abs_export_out = "${abs_exports_root_path}/${finalFilename}"
def export_filepath = new hudson.FilePath(channel, abs_export_out)
def export_out_folder = export_filepath.getParent()
def export_out_folder_string = export_out_folder.toString()
def export_out_file = export_filepath.getName()
echo "Creating folder ${export_out_folder_string}"
export_out_folder.mkdirs()
echo "Copying from ${pathToIntermediate} to ${abs_export_out}"
def intermediate_filepath = new hudson.FilePath(channel, pathToIntermediate)
intermediate_filepath.copyTo(export_filepath)
intermediate_filepath.delete()
}
// Given details about artifacts to import and details of installers to build, import artifacts that are required
def Map artifactsImporter(meta, artifactDetails, installerDetails) {
artifactFetched = [:]
@ -14,20 +38,58 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) {
return artifactFetched
}
// Export any required binaries to the <WORKSPACE>/bin folder
def placeBinaries(artifactDetails, installerDetails) {
for (installer in toBuild) {
(installertool, artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer)
if (exportBinaries) {
(sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactSet)
for (binaryFolder in binariesFolders) {
def binariesSourceFolder = "${destFolder}/${binaryFolder}"
def binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}"
echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}"
fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)])
@NonCPS
def checkISMfilebackend(String xml) {
def String message = ""
def parsedxml = new XmlParser(true, true, true).parseText(xml)
def pathvariableinfo = [:]
def exitcode = 0
// Load up a map of path variable and path pairs
// Ignore blank path definitions
// Ignore the workspace definition which should have the only absolute path
parsedxml.each{ table ->
if (table.@name == "ISPathVariable") {
table['row'].each{ row ->
def pathname = row['td'][0].value()[0]
def path = row['td'][1].value()[0]
if (path && pathname != 'WORKSPACE') {
pathvariableinfo[pathname] = path
}
}
}
}
// Check that each path variable is defined beginning with a path variable or the WORKSPACE one
pathvariableinfo.each{ pair ->
message += "pair key: ${pair.key} value: ${pair.value}\n"
def path = pair.value
def pathvarmatcher = path =~ /<(.*)>.*/
if (pathvarmatcher.matches()) {
def pathvarfound = pathvarmatcher.group(1)
if (pathvarfound != 'WORKSPACE') {
if (!pathvariableinfo.containsKey(pathvarfound)) {
message += "Undefined path variable ${pathvarfound} found\n"
exitcode = 1
}
}
} else {
message += "Path: ${path} does not begin with a path variable\n"
exitcode = 1
}
}
return [exitcode, message]
}
return this;
def checkISMfile(String ISMfile) {
echo "Checking ISM file: ${ISMfile}"
def xml = readFile(ISMfile)
(exitcode, message) = checkISMfilebackend(xml)
echo message
if (exitcode > 0) {
error "ISM file errors found"
} else {
echo "Checked ISM file - ok"
}
}
return this;

View file

@ -233,6 +233,7 @@ class ReleaseInfo extends GISJsonSerialised {
}
void addReleaseItem(String releaseitem) {
script.echo("Adding release item: ${releaseitem} to release items")
releaseitems.add(releaseitem)
}
@ -490,6 +491,7 @@ class MetaInfo extends GISJsonSerialised {
}
void markreleased() {
script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released")
releaseinfo.markreleased()
for (associated in associatedmeta) {
associated.meta.releaseinfo.markreleased()
@ -569,6 +571,7 @@ class MetaInfo extends GISJsonSerialised {
}
def deserialiseJson(String filepath) {
script.echo("Deserialising JSON at file: ${filepath}")
def rawmeta = script.readJSON(file: filepath)
deserialiseJson(rawmeta)
}
@ -615,6 +618,8 @@ class MetaInfo extends GISJsonSerialised {
def importedmetafile = adjustedpath + "/" + metafilepath()
if (script.fileExists(importedmetafile)) {
mi.deserialiseJson(importedmetafile)
} else {
script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist")
}
RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi)
@ -655,6 +660,10 @@ class MetaInfo extends GISJsonSerialised {
return METAFOLDERNAME
}
static String difffolder() {
return DIFFFOLDERNAME
}
String generateartifacts(ArrayList<String> releasedfromhere = new ArrayList<String>(), // These items are recorded in meta as released from here
ArrayList<String> releasedfromsubmeta = new ArrayList<String>(), // These items are not recorded in meta as released from here
String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}"
@ -763,6 +772,11 @@ MetaInfo metareset(script) {
dir(mi.folder()) {
deleteDir()
}
dir(mi.difffolder()) {
deleteDir()
}
return mi
}

View file

@ -8,7 +8,6 @@ summaryfile = "${summaryfolder}/summary.txt"
summarystring = ""
releaserepofolder = "${WORKSPACE}/releaserepos"
filteredreleaserepofolder = "${WORKSPACE}/filteredreleaserepos"
tempfolder = "${WORKSPACE}/temp"
def resetfolders() {
dir(summaryfolder) {
@ -20,9 +19,6 @@ def resetfolders() {
dir(filteredreleaserepofolder) {
deleteDir()
}
dir(tempfolder) {
deleteDir()
}
}
def productuniquefolder(releasemeta) {
@ -61,8 +57,7 @@ def guessreporootsuffix(componentURL) {
}
gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy")
def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, itempathstopublish) {
// Copy relevant artifacts to temp folder
def getcomponentartifacts(releaseinfo, String repofolder, itempathstopublish) {
summaryappend("Artifacts: ")
def releasemeta = releaseinfo.meta
@ -83,7 +78,7 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder, ite
}
if (publishablepaths.size() > 0) {
def reporootsuffix = guessreporootsuffix(componentURL)
gh.gitCheckout(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone
gh.gitCheckoutAlt(componentURL, revision, repofolder, reporootsuffix, 1, true, publishablepaths) // Shallow clone
}
else {
summaryappend("None ")
@ -124,11 +119,10 @@ def copydirectory(File dirFrom, File dirTo) {
def recallcomponent(releaseinfo, String softwarerelease, itempathstopublish) {
def releasemeta = releaseinfo.meta
def String uniquefolderforrepo = uniqueabsoluterreleasefolder(releaserepofolder, softwarerelease, releasemeta)
def String uniquefoldertemp = uniqueabsoluterreleasefolder(tempfolder, softwarerelease, releasemeta)
summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ")
getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp, itempathstopublish)
getcomponentartifacts(releaseinfo, uniquefolderforrepo, itempathstopublish)
summaryappend("\n")
}

View file

@ -1,8 +1,25 @@
// The filepath must end in "<folder>/**" or "<folder>/" - Glob wildcard - as a git sparse checkout
def allowed(String filepath) {
if (filepath.endsWith('/**') || filepath.endsWith('/')) {
def files = findFiles glob: filepath
if (files.length > 0) {
return
} else {
error "There are no files/folders under the publishable item: ${filepath}"
}
}
error "The publishable item: ${filepath} does not end in <folder>/** or <folder>/ (for git sparse checkouts)"
}
// Generate a list of artifact files that are publishable to outside parties
def generatelist(meta, publishable) {
filelist = []
for (filepath in publishable) {
if (filepath != meta.METAARTIFACTS) {
allowed(filepath)
}
meta.addPublishable(filepath)
}
}

View file

@ -61,7 +61,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) {
}
previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder)
dir(checkoutfolder) {
// Clear all existing files
// Clear all existing files - Note that this does not remove .git*
sh("git rm -r *")
}
return previousreleasemeta
@ -92,9 +92,10 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) {
}
}
def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) {
def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) {
// Copy relevant artifacts to temp folder
summaryappend("Artifacts: ")
def boolean result = true
for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) {
def filterpath = artifactfilter
if (buildinfo.offsetpath) {
@ -102,22 +103,31 @@ def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, Strin
}
// Copy relevant artifacts to temp folder
echo "Copying artifacts: ${filterpath} to ${tempfolder}"
copyArtifacts(
filter: filterpath,
projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname,
selector: [
$class: 'ParameterizedBuildSelector',
parameterName: "<SpecificBuildSelector plugin='copyartifact@1.41'><buildNumber>${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}</buildNumber></SpecificBuildSelector>"
],
target: tempfolder
)
// Move relevant artifacts from temp folder to release position
def originalrootdir = tempfolder + "/" + buildinfo.offsetpath
dir(originalrootdir) {
sh("mv * ${repofolder}")
try {
copyArtifacts(
filter: filterpath,
projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname,
selector: [
$class: 'ParameterizedBuildSelector',
parameterName: "<SpecificBuildSelector plugin='copyartifact@1.41'><buildNumber>${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}</buildNumber></SpecificBuildSelector>"
],
target: tempfolder
)
} catch (Exception e) {
echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}"
result = false
summaryappend("Does not exist:")
}
if (result) {
// Move relevant artifacts from temp folder to release position
def originalrootdir = tempfolder + "/" + buildinfo.offsetpath
dir(originalrootdir) {
sh("mv * ${repofolder}")
}
}
summaryappend("${artifactfilter},")
}
return result
}
def releasegitadd() {
@ -189,7 +199,7 @@ def boolean releasegittag(String postcommitshacode, String tag) {
if (existingshacode) { // Tag exists
if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD
// This is a problem - The tag is already at another position in git repo
setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it")
setunstable("The release tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it")
summaryappend("(${tag} - Could not add!)")
} else {
summaryappend("(${tag} - Already added)")
@ -275,6 +285,22 @@ def sourcegitpush(String dryrunpush ) {
}
}
def boolean isalreadyreleased(buildmeta, tag) {
def boolean alreadyreleased = buildmeta.releaseinfo.released
if (alreadyreleased) {
echo "Component is already marked as released: ${alreadyreleased}"
} else {
String commitshacode = releasegitgetshacode(tag)
if (commitshacode) {
echo "Tag: ${tag} already exists in this repo therefore marking as already released"
alreadyreleased = true
} else {
alreadyreleased = false
}
}
return alreadyreleased
}
def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) {
summaryappend(" Git operations: ")
def String buildstamptag = buildmeta.releasetag()
@ -283,8 +309,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source
def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim()
echo "Pre commit sha code: ${precommitshacode}"
def boolean alreadyreleased = buildmeta.releaseinfo.released
echo "Already released: ${alreadyreleased}"
def boolean alreadyreleased = isalreadyreleased(buildmeta, buildstamptag)
def String postcommitshacode
def boolean changed = false
@ -342,15 +367,15 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri
checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo)
getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)
gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush)
if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) {
gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush)
}
summaryappend("\n")
}
def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) {
int md = maxdepth.toInteger()
def int md = maxdepth.toInteger()
resetfolders()
preconditions()

View file

@ -4,7 +4,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy")
// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data
def registerrepo(meta, String releaserepo, String repoRootSuffix = "") {
if (releaserepo?.trim()) {
vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo", repoRootSuffix)
vc.checkRepoExists(releaserepo)
}
meta.releaseinfo.set(releaserepo)
}

View file

@ -11,12 +11,12 @@ String getGitInfo(String workingSetRootPath) {
}
@NonCPS
def SparseCheckoutPaths sparsecheckoutconfig(ArrayList<String> paths) {
def ArrayList<SparseCheckoutPath> sparsecheckoutconfig(ArrayList<String> paths) {
ArrayList<SparseCheckoutPath> sparsecheckoutpaths = new ArrayList<SparseCheckoutPath>()
for (path in paths) {
sparsecheckoutpaths.add(new SparseCheckoutPath(path))
}
return new SparseCheckoutPaths(sparsecheckoutpaths)
return sparsecheckoutpaths
}
def String uniquereporootname(String gitURL) {
@ -75,8 +75,90 @@ def String checkouttoreporoot(String gitURL) {
return reporootsubfolder
}
def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList<String> sparseitemlist = new ArrayList<String>()) {
def reporootpath = checkouttoreporoot(gitURL)
def checkgitlfsinstall() {
if (isUnix()) {
sh """
git lfs install
if [ \$? -ne 0 ]; then exit 1; fi
"""
} else {
bat """
git lfs install
if errorlevel 1 EXIT /b 1
"""
}
}
//No credentials specified
//Cloning the remote Git repository
//Cloning repository git@hydra:print-server-releases/print_server_whole.git
//> git init /home/jenkins_user/jenkins/jobs/Print_Server_publish_to_customer/ws/whole # timeout=10
//Using reference repository: /home/jenkins_user/reporoot/git_hydra_print_server_releases_print_server_whole_git
//Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git
// > git --version # timeout=10
// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120
// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10
// > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10
// > git config remote.origin.url git@hydra:print-server-releases/print_server_whole.git # timeout=10
// Cleaning workspace
// > git rev-parse --verify HEAD # timeout=10
// No valid HEAD. Skipping the resetting
// > git clean -fdx # timeout=10
// Fetching upstream changes from git@hydra:print-server-releases/print_server_whole.git
// > git fetch --tags --progress git@hydra:print-server-releases/print_server_whole.git +refs/heads/*:refs/remotes/origin/* # timeout=120
// > git rev-parse refs/tags/2.10180.7389.21125_7^{commit} # timeout=10
// > git rev-parse refs/remotes/origin/refs/tags/2.10180.7389.21125_7^{commit} # timeout=10
// Checking out Revision 64cac35e9a7603c060135f61f91b3f809bbdf160 (refs/tags/2.10180.7389.21125_7)
// Enabling Git LFS pull
// > git config core.sparsecheckout # timeout=10
// > git checkout -f 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=120
// > git config --get remote.origin.url # timeout=10
// > git lfs pull origin # timeout=120
// Commit message: "2.10180.7389.21125_7"
// > git rev-list --no-walk 64cac35e9a7603c060135f61f91b3f809bbdf160 # timeout=10
// > git rev-parse --is-inside-work-tree # timeout=10
// Fetching changes from the remote Git repository
// > git config remote.origin.url git@hydra:releases/print-server-installer.git # timeout=10
// Cleaning workspace
// > git rev-parse --verify HEAD # timeout=10
// Resetting working tree
// > git reset --hard # timeout=10
// > git clean -fdx # timeout=10
// Using shallow fetch with depth 1
// Fetching upstream changes from git@hydra:releases/print-server-installer.git
// > git --version # timeout=10
// > git fetch --tags --progress git@hydra:releases/print-server-installer.git +refs/heads/*:refs/remotes/origin/* --depth=1 # timeout=120
// > git rev-parse refs/remotes/origin/master^{commit} # timeout=10
// > git rev-parse refs/remotes/origin/origin/master^{commit} # timeout=10
// Checking out Revision d0788d22622862e441c5995a22629846573c39f3 (refs/remotes/origin/master)
// Enabling Git LFS pull
// > git config core.sparsecheckout # timeout=10
// > git read-tree -mu HEAD # timeout=10
// > git checkout -f d0788d22622862e441c5995a22629846573c39f3 # timeout=120
// > git config --get remote.origin.url # timeout=10
// > git lfs pull origin # timeout=120
// Commit message: "2.10180.7389.21125_3"
// > git rev-list --no-walk d0788d22622862e441c5995a22629846573c39f3 # timeout=10
def String lfsinstallandreporoot(String gitURL) {
checkgitlfsinstall()
return checkouttoreporoot(gitURL)
}
def showsparseitemlist(ArrayList<String> sparseitemlist = new ArrayList<String>()) {
if (!sparseitemlist.isEmpty()) {
def itemstring = "Items in sparse item list:\n"
for (item in sparseitemlist) {
itemstring += item + "\n"
}
echo itemstring
}
}
def docheckout(String gitURL, String revision, int cloneDepth, String reporootpath, boolean shallowClone, String targetDir, sparseitemlist) {
checkout([
$class: 'GitSCM',
userRemoteConfigs: [
@ -92,52 +174,176 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo
[$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120],
[$class: 'CheckoutOption', timeout: 120],
[$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir],
sparsecheckoutconfig(sparseitemlist)
[$class: 'SparseCheckoutPaths', sparseCheckoutPaths: sparsecheckoutconfig(sparseitemlist)],
],
submoduleCfg: [],
])
}
// gitCheckout (below) cannot handle sparse checkout and lfs at the moment - see https://issues.jenkins-ci.org/browse/JENKINS-43427 - see gitCheckoutAlt()
def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList<String> sparseitemlist = new ArrayList<String>()) {
def String reporootpath = lfsinstallandreporoot(gitURL)
showsparseitemlist(sparseitemlist)
docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist)
}
def gitinit(String targetDir, String reporootpath) {
if (!fileExists("${targetDir}/.git")) { // Git folder does not already exist
echo "git init ${targetDir}"
if (isUnix()) {
sh """
git init "${targetDir}"
"""
} else {
bat """
git init "${targetDir}"
"""
}
}
echo "Writing ${reporootpath}/.git/objects to ${targetDir}/.git/objects/info/alternates"
dir("${targetDir}/.git/objects/info") {
writeFile(file: 'alternates', text: "${reporootpath}/.git/objects")
}
}
def setlfsfetchinclude(String targetDir, ArrayList<String> sparseitemlist) {
def String sparseitemliststring = ""
if (!sparseitemlist.isEmpty()) {
for (item in sparseitemlist) {
sparseitemliststring += "${item},"
}
echo "git config lfs.fetchinclude ${sparseitemliststring}"
dir(targetDir) {
if (isUnix()) {
sh """
git config lfs.fetchinclude "${sparseitemliststring}"
"""
} else {
bat """
git config lfs.fetchinclude "${sparseitemliststring}"
"""
}
}
}
}
def gitsparserestore(String targetDir) {
echo "Clean out full fetch; Force a sparse fetch"
dir(targetDir) {
if (isUnix()) {
sh """
git rm -rf . > /dev/null
git clean -fxd > /dev/null
git reset --hard > /dev/null
"""
} else {
bat """
git rm -rf . 2> nul
git clean -fxd 2> nul
git reset --hard 2> nul
"""
}
}
}
def List convertSparseItemsString(String itemstopublish) {
def List sparsepaths = []
boolean nonempty = itemstopublish?.trim()
if (nonempty) {
def String[] items = itemstopublish.trim().split(',')
def String allpaths = ""
for(String item : items) {
sparsepaths.add(item)
allpaths += "${item}\n"
}
echo "Selected item to publish:\n${allpaths}"
}
return sparsepaths
}
def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList<String> sparseitemlist = new ArrayList<String>()) {
echo "Performing alternative checkout"
def String reporootpath = lfsinstallandreporoot(gitURL)
showsparseitemlist(sparseitemlist)
if (!sparseitemlist.isEmpty()) {
gitinit(targetDir, reporootpath)
setlfsfetchinclude(targetDir, sparseitemlist)
}
docheckout(gitURL, revision, cloneDepth, reporootpath, shallowClone, targetDir, sparseitemlist)
if (!sparseitemlist.isEmpty()) {
gitsparserestore(targetDir)
}
}
def checkRepoExists(String gitURL) {
def int result = 0
if (isUnix()) {
result = sh( script: """
git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY
""",
returnStatus: true
)
} else {
result = bat( script: """
@git ls-remote "${gitURL}" CHECK_GIT_REMOTE_URL_REACHABILITY
""",
returnStatus: true
)
}
echo "git ls-remote return result: ${result}"
if (result > 0) {
error("The release repo: ${gitURL} cannot be found. Please ensure this exists before building")
} else {
echo "The release repo: ${gitURL} exists"
}
}
String getSHAFromVersion(String versionFolder) {
dir (versionFolder) {
dir (versionFolder) {
srcRevision = readFile("gitrevision.txt").trim()
}
return srcRevision
}
return srcRevision
}
boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) {
def keys = versions.keySet()
if (keys.size() < 1) {
return false
}
if (keys.size() == 1) {
return true
}
if (keys.size() < 1) {
return false
}
if (keys.size() == 1) {
return true
}
else {
version = ""
for (key in keys) {
nextversion = versions.get(key)
if (version == "") {
version = nextversion
}
if (nextversion != version) {
return false
}
}
}
return true
version = ""
for (key in keys) {
nextversion = versions.get(key)
if (version == "") {
version = nextversion
}
if (nextversion != version) {
return false
}
}
}
return true
}
def stripgithistory(String rootfolder) {
echo "Stripping git history out of ${rootfolder}"
dir(rootfolder) {
sh """#!/bin/bash
pwd
find . -type d -name ".git" | xargs -iXX rm -rf XX
find . -name ".gitignore" | xargs -iXX rm -rf XX
find . -name ".gitmodules" | xargs -iXX rm -rf XX
find . -iname ".gitattributes" | xargs -iXX rm -rf XX
"""
if (isUnix()) {
sh """#!/bin/bash
pwd
find . -type d -name ".git" | xargs -iXX rm -rf XX
find . -name ".gitignore" | xargs -iXX rm -rf XX
find . -name ".gitmodules" | xargs -iXX rm -rf XX
find . -iname ".gitattributes" | xargs -iXX rm -rf XX
find . -iname "*@tmp" | xargs -iXX rm -rf XX
"""
} else {
echo "Not supporting git stripping in Windows at the moment"
}
}
}