277 lines
11 KiB
Python
277 lines
11 KiB
Python
# Create a release from several incoming component sources
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import stat
|
|
|
|
# Source name Source Workspace location
|
|
sourceItems = [
|
|
( "Build (compile) artifacts", "imports/build/version", ),
|
|
]
|
|
|
|
releaseRepo = os.environ["RELEASE_REPO"]
|
|
|
|
releaseRepos = [ releaseRepo,
|
|
]
|
|
|
|
releaseRepoDir = os.path.splitext(os.path.basename(releaseRepo))
|
|
releaseRepoPath = releaseRepoDir[0]+"/"
|
|
|
|
# Relative to "releases" - Note the base directories are created by the release repo cloning
|
|
makePaths = [ releaseRepoDir[0],
|
|
]
|
|
|
|
# List of file suffixes to ignore when copying a tree
|
|
copyIgnorePatterns = { "no_pdb" : ("*.pdb", "*.exp",),
|
|
"no_exedll" : ("*.exe", "*.dll", "*.exe.config", "*.dll.config", "*.xml", "*.XML", "*.bmp", "*.txt",),
|
|
}
|
|
|
|
# Source (relative to workspace) Destination (relative to "releases") Copy ignore pattern
|
|
copyPaths = [ #( "imports/install/installers/install_pdf/x64/", "global-graphics-pdf/x64/", None ),
|
|
# ( "imports/install/bin/LayoutSet/bin/x64/Release/", "layout/bin/x64/", "no_pdb" ),
|
|
( "imports/build/", releaseRepoPath, None ),
|
|
|
|
]
|
|
|
|
releaseRepo = os.environ["RELEASE_REPO"]
|
|
workspace = os.environ["WORKSPACE"]
|
|
releaseCommit = os.environ["RELEASE_COMMIT"].strip().lower()
|
|
dryRunPush = os.environ["DRY_RUN_PUSH"].strip().lower()
|
|
gitExecutable = "git"
|
|
|
|
def isCommitRequired():
|
|
commitRequired = False
|
|
if releaseCommit == "true":
|
|
commitRequired = True
|
|
return commitRequired
|
|
|
|
class RevisionError(Exception):
|
|
def __init__(self, componentName):
|
|
self._componentName = componentName
|
|
def component(self):
|
|
return self._componentName
|
|
|
|
versionPath = "version"
|
|
gitRevisionPath = "gitrevision.txt"
|
|
def allGitRevisionsMatch():
|
|
match = False
|
|
gitRevision = ""
|
|
for name, sourceLocation in sourceItems:
|
|
gitRevisionFile = os.path.join(workspace, sourceLocation, gitRevisionPath)
|
|
grf = open(gitRevisionFile)
|
|
fileGitRevision = grf.readline().strip().lower()
|
|
grf.close()
|
|
print("Component: ", name, " has git revision: ", fileGitRevision)
|
|
if fileGitRevision == "":
|
|
raise RevisionError(name)
|
|
if gitRevision == "":
|
|
gitRevision = fileGitRevision
|
|
match = True
|
|
else:
|
|
if gitRevision != fileGitRevision:
|
|
match = False
|
|
break
|
|
return match
|
|
|
|
# The definitive version info is stored within the version import
|
|
|
|
def getVersionNumber():
|
|
versionFilePath = os.path.join(workspace, sourceItems[0][1], "buildstamp.txt")
|
|
rp = open(versionFilePath)
|
|
versionNumber = rp.readline().strip().lower()
|
|
rp.close()
|
|
print("versionNumber: ",versionNumber)
|
|
return versionNumber
|
|
|
|
class RepoError(Exception):
|
|
def __init__(self, componentName):
|
|
self._componentName = componentName
|
|
def component(self):
|
|
return self._componentName
|
|
|
|
def removeReadonly(func, path, _):
|
|
"Clear the readonly bit and reattempt the removal"
|
|
os.chmod(path, stat.S_IWRITE)
|
|
func(path)
|
|
|
|
releasesRoot = "releases"
|
|
releasesBranch = "master"
|
|
def createReleaseFolders():
|
|
releasesRootPath = os.path.join(workspace, releasesRoot)
|
|
# Delete the release root path if it exists and create a new empty one
|
|
if os.path.isdir(releasesRootPath):
|
|
shutil.rmtree(releasesRootPath, onerror=removeReadonly)
|
|
|
|
os.makedirs(releasesRootPath)
|
|
curdir = os.getcwd()
|
|
os.chdir(releasesRootPath)
|
|
print("Changing directory from: ", curdir, " to: ", releasesRootPath)
|
|
try:
|
|
for releaseGitRepo in releaseRepos:
|
|
# Now git checkout (-n) each relevant repository
|
|
print("Cloning git repository: ", releaseGitRepo)
|
|
# Git clone
|
|
spout = subprocess.check_output([gitExecutable, "clone", "-n", "-b", releasesBranch, "--depth", "1", releaseGitRepo], stderr = subprocess.STDOUT)
|
|
print("Output:", spout.decode('utf-8'))
|
|
except:
|
|
print("Exception occurred while cloning: ", releaseGitRepo, " Exception: ", sys.exc_info()[0])
|
|
os.chdir(curdir)
|
|
return False;
|
|
|
|
os.chdir(curdir)
|
|
return True;
|
|
|
|
def makeDestinationPaths():
|
|
try:
|
|
for relativeMakePath in makePaths:
|
|
makePath = os.path.join(workspace, releasesRoot, relativeMakePath)
|
|
print("makePath is ",makePath)
|
|
os.makedirs(makePath, exist_ok = True)
|
|
except:
|
|
print("Exception occurred while making path: ", makePath, " Exception: ", sys.exc_info()[0])
|
|
return False
|
|
|
|
return True
|
|
|
|
def copytree(src, dst, symlinks = False, ignore = None):
|
|
if not os.path.exists(dst):
|
|
os.makedirs(dst)
|
|
shutil.copystat(src, dst)
|
|
lst = os.listdir(src)
|
|
if ignore:
|
|
excl = ignore(src, lst)
|
|
lst = [x for x in lst if x not in excl]
|
|
for item in lst:
|
|
s = os.path.join(src, item)
|
|
d = os.path.join(dst, item)
|
|
if symlinks and os.path.islink(s):
|
|
if os.path.lexists(d):
|
|
os.remove(d)
|
|
os.symlink(os.readlink(s), d)
|
|
try:
|
|
st = os.lstat(s)
|
|
mode = stat.S_IMODE(st.st_mode)
|
|
os.lchmod(d, mode)
|
|
except:
|
|
pass # lchmod not available
|
|
elif os.path.isdir(s):
|
|
copytree(s, d, symlinks, ignore)
|
|
else:
|
|
shutil.copy2(s, d)
|
|
|
|
def copySourceToDestination():
|
|
# Copy from source tree to destination tree and don't copy any files that are identified by the filter
|
|
try:
|
|
for relativeSourcePath, relativeDestinationPath, filterName in copyPaths:
|
|
sourcePath = os.path.join(workspace, relativeSourcePath)
|
|
destinationPath = os.path.join(workspace, releasesRoot, relativeDestinationPath)
|
|
if filterName:
|
|
print("Copying from: ", sourcePath, " to: ", destinationPath, " using filter:", filterName)
|
|
ignoreFilter = copyIgnorePatterns[filterName]
|
|
copytree(sourcePath, destinationPath, ignore=shutil.ignore_patterns(*ignoreFilter))
|
|
else:
|
|
print("Copying from: ", sourcePath, " to: ", destinationPath)
|
|
copytree(sourcePath, destinationPath)
|
|
except:
|
|
print("Exception occurred while copying from: ", sourcePath, " to: ", destinationPath, "Exception: ", sys.exc_info()[0])
|
|
return False
|
|
|
|
return True
|
|
import re
|
|
|
|
# eg. "git@hydra:releases/atlas-layout.git" is converted to "<workspace>/releases/atlas-layout"
|
|
def getRepositoryPath(repositoryUrl):
|
|
releasesRootPath = os.path.join(workspace, releasesRoot)
|
|
pattern = re.compile('.*@.*:.*/([\w-]+)\.git')
|
|
m = pattern.match(repositoryUrl)
|
|
return os.path.join(releasesRootPath, m.group(1))
|
|
|
|
def gitAddCommitTagAll(versionNumber):
|
|
curdir = os.getcwd()
|
|
try:
|
|
for releaseGitRepo in releaseRepos:
|
|
localRepoPath = getRepositoryPath(releaseGitRepo)
|
|
os.chdir(localRepoPath)
|
|
print("**************************")
|
|
# Git add
|
|
print("git adding files in local repo: ", localRepoPath)
|
|
spout = subprocess.check_output([gitExecutable, "add", "."], stderr = subprocess.STDOUT)
|
|
print("Result: ", spout.decode('utf-8'))
|
|
# Git status - to see any commit operation is required
|
|
spout = subprocess.check_output([gitExecutable, "status", "--porcelain"], stderr = subprocess.STDOUT)
|
|
if spout: # String is non-empty if repository is not up to date
|
|
# Git commit
|
|
commitComment = versionNumber
|
|
if "COMMIT_COMMENT" in os.environ :
|
|
import unicodedata
|
|
commitComment = commitComment + ' :: ' + "".join(ch for ch in os.environ["COMMIT_COMMENT"] if unicodedata.category(ch)[0]!="C") # Cleans up non printable characters
|
|
print("git committing local repo: ", localRepoPath, " Comment: ", commitComment)
|
|
spout = subprocess.check_output([gitExecutable, "commit", "-a", "-m", commitComment], stderr = subprocess.STDOUT)
|
|
print("Result: ", spout.decode('utf-8'))
|
|
else:
|
|
print("Not committing local repo: ", localRepoPath, " Repo is up to date / There are no changes.")
|
|
# Git tag
|
|
print("git tagging: in local repo: ", localRepoPath, " Tag: ", versionNumber)
|
|
spout = subprocess.check_output([gitExecutable, "tag", versionNumber], stderr = subprocess.STDOUT)
|
|
print("Result: ", spout.decode('utf-8'))
|
|
except:
|
|
print("Exception occurred while adding/committing/tagging from: ", localRepoPath, "Exception: ", sys.exc_info()[0])
|
|
os.chdir(curdir)
|
|
return False;
|
|
|
|
os.chdir(curdir)
|
|
return True;
|
|
|
|
def gitPushAll(versionNumber):
|
|
curdir = os.getcwd()
|
|
try:
|
|
for releaseGitRepo in releaseRepos:
|
|
localRepoPath = getRepositoryPath(releaseGitRepo)
|
|
os.chdir(localRepoPath)
|
|
print("**************************")
|
|
# Git push
|
|
print("git push local repo: ", localRepoPath)
|
|
if dryRunPush == "true":
|
|
spout = subprocess.check_output([gitExecutable, "push", "-n", "--tags", "origin", "master"], stderr = subprocess.STDOUT)
|
|
else:
|
|
spout = subprocess.check_output([gitExecutable, "push", "--tags", "origin", "master"], stderr = subprocess.STDOUT)
|
|
print("Result: ", spout.decode('utf-8'))
|
|
except:
|
|
print("Exception occurred while pushing from: ", localRepoPath, "Exception: ", sys.exc_info()[0])
|
|
os.chdir(curdir)
|
|
return False;
|
|
|
|
os.chdir(curdir)
|
|
return True;
|
|
|
|
def main():
|
|
if not isCommitRequired():
|
|
print("Commit has not been demanded - exiting")
|
|
sys.exit(0)
|
|
|
|
versionNumber = getVersionNumber()
|
|
print("Version number of release is: ", versionNumber)
|
|
|
|
if not createReleaseFolders():
|
|
print("The release folders could not be created")
|
|
sys.exit(1)
|
|
|
|
if not copySourceToDestination():
|
|
print("Failed to copy source to destination")
|
|
sys.exit(1)
|
|
|
|
if not gitAddCommitTagAll(versionNumber):
|
|
print("Failed to perform git add/commit/tag operation\n Determine git error cause and start this job again.")
|
|
sys.exit(1)
|
|
|
|
if not gitPushAll(versionNumber):
|
|
print("Failed to perform git push operation\n Determine git error cause. You may have to complete any other pushes by hand")
|
|
sys.exit(1)
|
|
|
|
sys.exit(0)
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|
|
|