firmware_tools/release_to_git/conditional_add.py

277 lines
11 KiB
Python

# Create a release from several incoming component sources
import os
import shutil
import subprocess
import sys
import stat
# Source name Source Workspace location
sourceItems = [
( "Build (compile) artifacts", "imports/build/version", ),
]
releaseRepo = os.environ["RELEASE_REPO"]
releaseRepos = [ releaseRepo,
]
releaseRepoDir = os.path.splitext(os.path.basename(releaseRepo))
releaseRepoPath = releaseRepoDir[0]+"/"
# Relative to "releases" - Note the base directories are created by the release repo cloning
makePaths = [ releaseRepoDir[0],
]
# List of file suffixes to ignore when copying a tree
copyIgnorePatterns = { "no_pdb" : ("*.pdb", "*.exp",),
"no_exedll" : ("*.exe", "*.dll", "*.exe.config", "*.dll.config", "*.xml", "*.XML", "*.bmp", "*.txt",),
}
# Source (relative to workspace) Destination (relative to "releases") Copy ignore pattern
copyPaths = [ #( "imports/install/installers/install_pdf/x64/", "global-graphics-pdf/x64/", None ),
# ( "imports/install/bin/LayoutSet/bin/x64/Release/", "layout/bin/x64/", "no_pdb" ),
( "imports/build/", releaseRepoPath, None ),
]
releaseRepo = os.environ["RELEASE_REPO"]
workspace = os.environ["WORKSPACE"]
releaseCommit = os.environ["RELEASE_COMMIT"].strip().lower()
dryRunPush = os.environ["DRY_RUN_PUSH"].strip().lower()
gitExecutable = "git"
def isCommitRequired():
commitRequired = False
if releaseCommit == "true":
commitRequired = True
return commitRequired
class RevisionError(Exception):
def __init__(self, componentName):
self._componentName = componentName
def component(self):
return self._componentName
versionPath = "version"
gitRevisionPath = "gitrevision.txt"
def allGitRevisionsMatch():
match = False
gitRevision = ""
for name, sourceLocation in sourceItems:
gitRevisionFile = os.path.join(workspace, sourceLocation, gitRevisionPath)
grf = open(gitRevisionFile)
fileGitRevision = grf.readline().strip().lower()
grf.close()
print("Component: ", name, " has git revision: ", fileGitRevision)
if fileGitRevision == "":
raise RevisionError(name)
if gitRevision == "":
gitRevision = fileGitRevision
match = True
else:
if gitRevision != fileGitRevision:
match = False
break
return match
# The definitive version info is stored within the version import
def getVersionNumber():
versionFilePath = os.path.join(workspace, sourceItems[0][1], "buildstamp.txt")
rp = open(versionFilePath)
versionNumber = rp.readline().strip().lower()
rp.close()
print("versionNumber: ",versionNumber)
return versionNumber
class RepoError(Exception):
def __init__(self, componentName):
self._componentName = componentName
def component(self):
return self._componentName
def removeReadonly(func, path, _):
"Clear the readonly bit and reattempt the removal"
os.chmod(path, stat.S_IWRITE)
func(path)
releasesRoot = "releases"
releasesBranch = "master"
def createReleaseFolders():
releasesRootPath = os.path.join(workspace, releasesRoot)
# Delete the release root path if it exists and create a new empty one
if os.path.isdir(releasesRootPath):
shutil.rmtree(releasesRootPath, onerror=removeReadonly)
os.makedirs(releasesRootPath)
curdir = os.getcwd()
os.chdir(releasesRootPath)
print("Changing directory from: ", curdir, " to: ", releasesRootPath)
try:
for releaseGitRepo in releaseRepos:
# Now git checkout (-n) each relevant repository
print("Cloning git repository: ", releaseGitRepo)
# Git clone
spout = subprocess.check_output([gitExecutable, "clone", "-n", "-b", releasesBranch, "--depth", "1", releaseGitRepo], stderr = subprocess.STDOUT)
print("Output:", spout.decode('utf-8'))
except:
print("Exception occurred while cloning: ", releaseGitRepo, " Exception: ", sys.exc_info()[0])
os.chdir(curdir)
return False;
os.chdir(curdir)
return True;
def makeDestinationPaths():
try:
for relativeMakePath in makePaths:
makePath = os.path.join(workspace, releasesRoot, relativeMakePath)
print("makePath is ",makePath)
os.makedirs(makePath, exist_ok = True)
except:
print("Exception occurred while making path: ", makePath, " Exception: ", sys.exc_info()[0])
return False
return True
def copytree(src, dst, symlinks = False, ignore = None):
if not os.path.exists(dst):
os.makedirs(dst)
shutil.copystat(src, dst)
lst = os.listdir(src)
if ignore:
excl = ignore(src, lst)
lst = [x for x in lst if x not in excl]
for item in lst:
s = os.path.join(src, item)
d = os.path.join(dst, item)
if symlinks and os.path.islink(s):
if os.path.lexists(d):
os.remove(d)
os.symlink(os.readlink(s), d)
try:
st = os.lstat(s)
mode = stat.S_IMODE(st.st_mode)
os.lchmod(d, mode)
except:
pass # lchmod not available
elif os.path.isdir(s):
copytree(s, d, symlinks, ignore)
else:
shutil.copy2(s, d)
def copySourceToDestination():
# Copy from source tree to destination tree and don't copy any files that are identified by the filter
try:
for relativeSourcePath, relativeDestinationPath, filterName in copyPaths:
sourcePath = os.path.join(workspace, relativeSourcePath)
destinationPath = os.path.join(workspace, releasesRoot, relativeDestinationPath)
if filterName:
print("Copying from: ", sourcePath, " to: ", destinationPath, " using filter:", filterName)
ignoreFilter = copyIgnorePatterns[filterName]
copytree(sourcePath, destinationPath, ignore=shutil.ignore_patterns(*ignoreFilter))
else:
print("Copying from: ", sourcePath, " to: ", destinationPath)
copytree(sourcePath, destinationPath)
except:
print("Exception occurred while copying from: ", sourcePath, " to: ", destinationPath, "Exception: ", sys.exc_info()[0])
return False
return True
import re
# eg. "git@hydra:releases/atlas-layout.git" is converted to "<workspace>/releases/atlas-layout"
def getRepositoryPath(repositoryUrl):
releasesRootPath = os.path.join(workspace, releasesRoot)
pattern = re.compile('.*@.*:.*/([\w-]+)\.git')
m = pattern.match(repositoryUrl)
return os.path.join(releasesRootPath, m.group(1))
def gitAddCommitTagAll(versionNumber):
curdir = os.getcwd()
try:
for releaseGitRepo in releaseRepos:
localRepoPath = getRepositoryPath(releaseGitRepo)
os.chdir(localRepoPath)
print("**************************")
# Git add
print("git adding files in local repo: ", localRepoPath)
spout = subprocess.check_output([gitExecutable, "add", "."], stderr = subprocess.STDOUT)
print("Result: ", spout.decode('utf-8'))
# Git status - to see any commit operation is required
spout = subprocess.check_output([gitExecutable, "status", "--porcelain"], stderr = subprocess.STDOUT)
if spout: # String is non-empty if repository is not up to date
# Git commit
commitComment = versionNumber
if "COMMIT_COMMENT" in os.environ :
import unicodedata
commitComment = commitComment + ' :: ' + "".join(ch for ch in os.environ["COMMIT_COMMENT"] if unicodedata.category(ch)[0]!="C") # Cleans up non printable characters
print("git committing local repo: ", localRepoPath, " Comment: ", commitComment)
spout = subprocess.check_output([gitExecutable, "commit", "-a", "-m", commitComment], stderr = subprocess.STDOUT)
print("Result: ", spout.decode('utf-8'))
else:
print("Not committing local repo: ", localRepoPath, " Repo is up to date / There are no changes.")
# Git tag
print("git tagging: in local repo: ", localRepoPath, " Tag: ", versionNumber)
spout = subprocess.check_output([gitExecutable, "tag", versionNumber], stderr = subprocess.STDOUT)
print("Result: ", spout.decode('utf-8'))
except:
print("Exception occurred while adding/committing/tagging from: ", localRepoPath, "Exception: ", sys.exc_info()[0])
os.chdir(curdir)
return False;
os.chdir(curdir)
return True;
def gitPushAll(versionNumber):
curdir = os.getcwd()
try:
for releaseGitRepo in releaseRepos:
localRepoPath = getRepositoryPath(releaseGitRepo)
os.chdir(localRepoPath)
print("**************************")
# Git push
print("git push local repo: ", localRepoPath)
if dryRunPush == "true":
spout = subprocess.check_output([gitExecutable, "push", "-n", "--tags", "origin", "master"], stderr = subprocess.STDOUT)
else:
spout = subprocess.check_output([gitExecutable, "push", "--tags", "origin", "master"], stderr = subprocess.STDOUT)
print("Result: ", spout.decode('utf-8'))
except:
print("Exception occurred while pushing from: ", localRepoPath, "Exception: ", sys.exc_info()[0])
os.chdir(curdir)
return False;
os.chdir(curdir)
return True;
def main():
if not isCommitRequired():
print("Commit has not been demanded - exiting")
sys.exit(0)
versionNumber = getVersionNumber()
print("Version number of release is: ", versionNumber)
if not createReleaseFolders():
print("The release folders could not be created")
sys.exit(1)
if not copySourceToDestination():
print("Failed to copy source to destination")
sys.exit(1)
if not gitAddCommitTagAll(versionNumber):
print("Failed to perform git add/commit/tag operation\n Determine git error cause and start this job again.")
sys.exit(1)
if not gitPushAll(versionNumber):
print("Failed to perform git push operation\n Determine git error cause. You may have to complete any other pushes by hand")
sys.exit(1)
sys.exit(0)
if __name__ == "__main__":
sys.exit(main())