# Create a release from several incoming component sources import os import shutil import subprocess import sys import stat # Source name Source Workspace location sourceItems = [ ( "Build (compile) artifacts", "imports/build/version", ), ] releaseRepo = os.environ["RELEASE_REPO"] releaseRepos = [ releaseRepo, ] releaseRepoDir = os.path.splitext(os.path.basename(releaseRepo)) releaseRepoPath = releaseRepoDir[0]+"/" # Relative to "releases" - Note the base directories are created by the release repo cloning makePaths = [ releaseRepoDir[0], ] # List of file suffixes to ignore when copying a tree copyIgnorePatterns = { "no_pdb" : ("*.pdb", "*.exp",), "no_exedll" : ("*.exe", "*.dll", "*.exe.config", "*.dll.config", "*.xml", "*.XML", "*.bmp", "*.txt",), } # Source (relative to workspace) Destination (relative to "releases") Copy ignore pattern copyPaths = [ #( "imports/install/installers/install_pdf/x64/", "global-graphics-pdf/x64/", None ), # ( "imports/install/bin/LayoutSet/bin/x64/Release/", "layout/bin/x64/", "no_pdb" ), ( "imports/build/", releaseRepoPath, None ), ] releaseRepo = os.environ["RELEASE_REPO"] workspace = os.environ["WORKSPACE"] releaseCommit = os.environ["RELEASE_COMMIT"].strip().lower() dryRunPush = os.environ["DRY_RUN_PUSH"].strip().lower() gitExecutable = "git" def isCommitRequired(): commitRequired = False if releaseCommit == "true": commitRequired = True return commitRequired class RevisionError(Exception): def __init__(self, componentName): self._componentName = componentName def component(self): return self._componentName versionPath = "version" gitRevisionPath = "gitrevision.txt" def allGitRevisionsMatch(): match = False gitRevision = "" for name, sourceLocation in sourceItems: gitRevisionFile = os.path.join(workspace, sourceLocation, gitRevisionPath) grf = open(gitRevisionFile) fileGitRevision = grf.readline().strip().lower() grf.close() print("Component: ", name, " has git revision: ", fileGitRevision) if fileGitRevision == "": raise RevisionError(name) if gitRevision == "": gitRevision = fileGitRevision match = True else: if gitRevision != fileGitRevision: match = False break return match # The definitive version info is stored within the version import def getVersionNumber(): versionFilePath = os.path.join(workspace, sourceItems[0][1], "buildstamp.txt") rp = open(versionFilePath) versionNumber = rp.readline().strip().lower() rp.close() print("versionNumber: ",versionNumber) return versionNumber class RepoError(Exception): def __init__(self, componentName): self._componentName = componentName def component(self): return self._componentName def removeReadonly(func, path, _): "Clear the readonly bit and reattempt the removal" os.chmod(path, stat.S_IWRITE) func(path) releasesRoot = "releases" releasesBranch = "master" def createReleaseFolders(): releasesRootPath = os.path.join(workspace, releasesRoot) # Delete the release root path if it exists and create a new empty one if os.path.isdir(releasesRootPath): shutil.rmtree(releasesRootPath, onerror=removeReadonly) os.makedirs(releasesRootPath) curdir = os.getcwd() os.chdir(releasesRootPath) print("Changing directory from: ", curdir, " to: ", releasesRootPath) try: for releaseGitRepo in releaseRepos: # Now git checkout (-n) each relevant repository print("Cloning git repository: ", releaseGitRepo) # Git clone spout = subprocess.check_output([gitExecutable, "clone", "-n", "-b", releasesBranch, "--depth", "1", releaseGitRepo], stderr = subprocess.STDOUT) print("Output:", spout.decode('utf-8')) except: print("Exception occurred while cloning: ", releaseGitRepo, " Exception: ", sys.exc_info()[0]) os.chdir(curdir) return False; os.chdir(curdir) return True; def makeDestinationPaths(): try: for relativeMakePath in makePaths: makePath = os.path.join(workspace, releasesRoot, relativeMakePath) print("makePath is ",makePath) os.makedirs(makePath, exist_ok = True) except: print("Exception occurred while making path: ", makePath, " Exception: ", sys.exc_info()[0]) return False return True def copytree(src, dst, symlinks = False, ignore = None): if not os.path.exists(dst): os.makedirs(dst) shutil.copystat(src, dst) lst = os.listdir(src) if ignore: excl = ignore(src, lst) lst = [x for x in lst if x not in excl] for item in lst: s = os.path.join(src, item) d = os.path.join(dst, item) if symlinks and os.path.islink(s): if os.path.lexists(d): os.remove(d) os.symlink(os.readlink(s), d) try: st = os.lstat(s) mode = stat.S_IMODE(st.st_mode) os.lchmod(d, mode) except: pass # lchmod not available elif os.path.isdir(s): copytree(s, d, symlinks, ignore) else: shutil.copy2(s, d) def copySourceToDestination(): # Copy from source tree to destination tree and don't copy any files that are identified by the filter try: for relativeSourcePath, relativeDestinationPath, filterName in copyPaths: sourcePath = os.path.join(workspace, relativeSourcePath) destinationPath = os.path.join(workspace, releasesRoot, relativeDestinationPath) if filterName: print("Copying from: ", sourcePath, " to: ", destinationPath, " using filter:", filterName) ignoreFilter = copyIgnorePatterns[filterName] copytree(sourcePath, destinationPath, ignore=shutil.ignore_patterns(*ignoreFilter)) else: print("Copying from: ", sourcePath, " to: ", destinationPath) copytree(sourcePath, destinationPath) except: print("Exception occurred while copying from: ", sourcePath, " to: ", destinationPath, "Exception: ", sys.exc_info()[0]) return False return True import re # eg. "git@hydra:releases/atlas-layout.git" is converted to "/releases/atlas-layout" def getRepositoryPath(repositoryUrl): releasesRootPath = os.path.join(workspace, releasesRoot) pattern = re.compile('.*@.*:.*/([\w-]+)\.git') m = pattern.match(repositoryUrl) return os.path.join(releasesRootPath, m.group(1)) def gitAddCommitTagAll(versionNumber): curdir = os.getcwd() try: for releaseGitRepo in releaseRepos: localRepoPath = getRepositoryPath(releaseGitRepo) os.chdir(localRepoPath) print("**************************") # Git add print("git adding files in local repo: ", localRepoPath) spout = subprocess.check_output([gitExecutable, "add", "."], stderr = subprocess.STDOUT) print("Result: ", spout.decode('utf-8')) # Git status - to see any commit operation is required spout = subprocess.check_output([gitExecutable, "status", "--porcelain"], stderr = subprocess.STDOUT) if spout: # String is non-empty if repository is not up to date # Git commit commitComment = versionNumber if "COMMIT_COMMENT" in os.environ : import unicodedata commitComment = commitComment + ' :: ' + "".join(ch for ch in os.environ["COMMIT_COMMENT"] if unicodedata.category(ch)[0]!="C") # Cleans up non printable characters print("git committing local repo: ", localRepoPath, " Comment: ", commitComment) spout = subprocess.check_output([gitExecutable, "commit", "-a", "-m", commitComment], stderr = subprocess.STDOUT) print("Result: ", spout.decode('utf-8')) else: print("Not committing local repo: ", localRepoPath, " Repo is up to date / There are no changes.") # Git tag print("git tagging: in local repo: ", localRepoPath, " Tag: ", versionNumber) spout = subprocess.check_output([gitExecutable, "tag", versionNumber], stderr = subprocess.STDOUT) print("Result: ", spout.decode('utf-8')) except: print("Exception occurred while adding/committing/tagging from: ", localRepoPath, "Exception: ", sys.exc_info()[0]) os.chdir(curdir) return False; os.chdir(curdir) return True; def gitPushAll(versionNumber): curdir = os.getcwd() try: for releaseGitRepo in releaseRepos: localRepoPath = getRepositoryPath(releaseGitRepo) os.chdir(localRepoPath) print("**************************") # Git push print("git push local repo: ", localRepoPath) if dryRunPush == "true": spout = subprocess.check_output([gitExecutable, "push", "-n", "--tags", "origin", "master"], stderr = subprocess.STDOUT) else: spout = subprocess.check_output([gitExecutable, "push", "--tags", "origin", "master"], stderr = subprocess.STDOUT) print("Result: ", spout.decode('utf-8')) except: print("Exception occurred while pushing from: ", localRepoPath, "Exception: ", sys.exc_info()[0]) os.chdir(curdir) return False; os.chdir(curdir) return True; def main(): if not isCommitRequired(): print("Commit has not been demanded - exiting") sys.exit(0) versionNumber = getVersionNumber() print("Version number of release is: ", versionNumber) if not createReleaseFolders(): print("The release folders could not be created") sys.exit(1) if not copySourceToDestination(): print("Failed to copy source to destination") sys.exit(1) if not gitAddCommitTagAll(versionNumber): print("Failed to perform git add/commit/tag operation\n Determine git error cause and start this job again.") sys.exit(1) if not gitPushAll(versionNumber): print("Failed to perform git push operation\n Determine git error cause. You may have to complete any other pushes by hand") sys.exit(1) sys.exit(0) if __name__ == "__main__": sys.exit(main())