From 8073420970773a084f2277544dd80114df854da9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Nov 2017 10:39:37 +0000 Subject: [PATCH 001/558] add README --- README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 From 744ee5f3d0d0fb0b9e873fba5c3e72105c962266 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Nov 2017 10:50:17 +0000 Subject: [PATCH 002/558] GIS-1621 - Add first jenkins groovy script --- ScriptTest/script_test_1.gsh | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 ScriptTest/script_test_1.gsh diff --git a/ScriptTest/script_test_1.gsh b/ScriptTest/script_test_1.gsh new file mode 100644 index 0000000..6b3e988 --- /dev/null +++ b/ScriptTest/script_test_1.gsh @@ -0,0 +1,75 @@ +pipeline { + // agent { label 'Pemba' } + // agent { label 'floreana' } + agent { label 'tiger' } + // agent any + + stages { + stage('Checkout') { + steps { + checkout([$class: 'GitSCM', + branches: [[name: 'refs/heads/master']], + doGenerateSubmoduleConfigurations: false, + extensions: [[$class: 'RelativeTargetDirectory', + relativeTargetDir: 'arepo'], + [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false, timeout: 60], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 60]], + submoduleCfg: [], + userRemoteConfigs: [[url: 'git@hydra:danhawkins/testproject.git']]]) + script { + if (isUnix()) { + sh 'echo $WORKSPACE' + sh 'ls -lsR $WORKSPACE' + } + else { + bat 'echo %WORKSPACE%' + bat 'dir "%WORKSPACE%"' + } + } + } + } + stage('Build') { + steps { + script { + if (isUnix()) { + sh 'echo make $APARAMETER' + sh 'echo $WORKSPACE' + sh 'cat $WORKSPACE/arepo/README2.md' + } + else { + bat 'echo make "%APARAMETER%"' + bat 'echo %WORKSPACE%' + bat 'type "%WORKSPACE%\\arepo\\README2.md"' + } + } + } + } + stage('Test'){ + steps { + script { + if (isUnix()) { + sh 'echo make check' + sh 'echo do test' + } + else { + bat 'echo make check' + bat 'echo do test' + } + } + } + } + stage('Deploy') { + steps { + script { + if (isUnix()) { + sh 'echo make publish' + } + else { + bat 'echo make publish' + } + } + } + } + } +} From 8d0a7bbd170fef8f97021fa24b6ee7c307effa49 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Nov 2017 17:22:37 +0000 Subject: [PATCH 003/558] GIS-1621 - Now working --- ScriptTest/script_test_1.gsh | 75 ------------ build/build.gsh | 222 +++++++++++++++++++++++++++++++++++ 2 files changed, 222 insertions(+), 75 deletions(-) delete mode 100644 ScriptTest/script_test_1.gsh create mode 100644 build/build.gsh diff --git a/ScriptTest/script_test_1.gsh b/ScriptTest/script_test_1.gsh deleted file mode 100644 index 6b3e988..0000000 --- a/ScriptTest/script_test_1.gsh +++ /dev/null @@ -1,75 +0,0 @@ -pipeline { - // agent { label 'Pemba' } - // agent { label 'floreana' } - agent { label 'tiger' } - // agent any - - stages { - stage('Checkout') { - steps { - checkout([$class: 'GitSCM', - branches: [[name: 'refs/heads/master']], - doGenerateSubmoduleConfigurations: false, - extensions: [[$class: 'RelativeTargetDirectory', - relativeTargetDir: 'arepo'], - [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false, timeout: 60], - [$class: 'CleanBeforeCheckout'], - [$class: 'CheckoutOption', timeout: 60]], - submoduleCfg: [], - userRemoteConfigs: [[url: 'git@hydra:danhawkins/testproject.git']]]) - script { - if (isUnix()) { - sh 'echo $WORKSPACE' - sh 'ls -lsR $WORKSPACE' - } - else { - bat 'echo %WORKSPACE%' - bat 'dir "%WORKSPACE%"' - } - } - } - } - stage('Build') { - steps { - script { - if (isUnix()) { - sh 'echo make $APARAMETER' - sh 'echo $WORKSPACE' - sh 'cat $WORKSPACE/arepo/README2.md' - } - else { - bat 'echo make "%APARAMETER%"' - bat 'echo %WORKSPACE%' - bat 'type "%WORKSPACE%\\arepo\\README2.md"' - } - } - } - } - stage('Test'){ - steps { - script { - if (isUnix()) { - sh 'echo make check' - sh 'echo do test' - } - else { - bat 'echo make check' - bat 'echo do test' - } - } - } - } - stage('Deploy') { - steps { - script { - if (isUnix()) { - sh 'echo make publish' - } - else { - bat 'echo make publish' - } - } - } - } - } -} diff --git a/build/build.gsh b/build/build.gsh new file mode 100644 index 0000000..500f20b --- /dev/null +++ b/build/build.gsh @@ -0,0 +1,222 @@ +pipeline { + // agent { label 'Pemba' } + // agent { label 'floreana' } + agent { label 'build' } + // agent any + stages { + stage('Checkout source') { + steps { + checkout([$class: 'GitSCM', + branches: [ + [name: 'origin/${BRANCH_NAME}'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: 0, noTags: false, reference: '${REPO_ROOT}/newsw', shallow: false, timeout: 60], + [$class: 'CheckoutOption', timeout: 60], + [$class: 'RelativeTargetDirectory', relativeTargetDir: 'GitRepository'], + [$class: 'PathRestriction', excludedRegions: '', includedRegions: '''src/.* autobuild/.*'''] + ], + submoduleCfg: [ + ], + userRemoteConfigs: [ + [url: 'git@hydra:gis/newsw.git'] + ] + ]) + } + } + stage('Checkout dependencies') { + steps { + checkout([$class: 'GitSCM', + branches: [ + [name: 'refs/tags/${FD1200_FIRMWARE_RELEASE_VERSION}'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: 0, noTags: false, reference: '${REPO_ROOT}/samba_fw_releases', shallow: true, timeout: 60], + [$class: 'CheckoutOption', timeout: 60], + [$class: 'RelativeTargetDirectory', relativeTargetDir: 'samba_fw'] + ], + submoduleCfg: [ + ], + userRemoteConfigs: [ + [url: 'git@hydra:gis/samba_fw_releases.git'] + ] + ]) + checkout([$class: 'GitSCM', + branches: [ + [name: 'refs/tags/${XR_1003_EMBEDDED_LOGIC_RELEASE_VERSION}'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: 0, noTags: false, reference: '${REPO_ROOT}/HMB_XR_1003_Embeddded_Logic.git', shallow: true, timeout: 60], + [$class: 'CheckoutOption', timeout: 60], + [$class: 'RelativeTargetDirectory', relativeTargetDir: 'xr1003_el'] + ], + submoduleCfg: [ + ], + userRemoteConfigs: [ + [url: 'git@hydra:releases/HMB_XR_1003_Embeddded_Logic.git'] + ] + ]) + checkout([$class: 'GitSCM', + branches: [ + [name: 'refs/tags/${VERSION_STAMPING_VERSION}'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: false, reference: '${REPO_ROOT}\\ver_stamping_releases', shallow: true, timeout: 60], + [$class: 'CheckoutOption', timeout: 60], + [$class: 'RelativeTargetDirectory', relativeTargetDir: 'ver_stamping'] + ], + submoduleCfg: [], + userRemoteConfigs: [ + [url: 'git@hydra:releases/ver_stamping_releases.git'] + ] + ]) + checkout([$class: 'GitSCM', + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', noTags: false, reference: '${REPO_ROOT}\\print_release_difference', shallow: true, timeout: 60], + [$class: 'CheckoutOption', timeout: 60], + [$class: 'RelativeTargetDirectory', relativeTargetDir: 'print_release_difference'] + ], + submoduleCfg: [ + ], + userRemoteConfigs: [ + [url: 'git@hydra:gis/print_release_difference.git'] + ] + ]) + + } + } + stage('Setup links') { + steps { + bat ''' + @echo on' + rmdir jobscripts + mklink /J jobscripts GitRepository\\autobuild\\build + rmdir jobscripts_common + mklink /J jobscripts_common "GitRepository\\autobuild\\common" + rmdir security + mklink /J security "GitRepository\\autobuild\\security" + rmdir src + mklink /J src GitRepository\\src + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Imports and exports') { + steps { + bat ''' + call jobscripts\\reset_imports_and_exports.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Clean') { + steps { + bat ''' + call jobscripts\\clean.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Generating version number') { + steps { + bat ''' + call jobscripts\\versioning.bat + if errorlevel 1 exit /b 1 + ''' + script { + env.BUILD_STAMP = readFile('version/buildstamp.txt').trim() + env.SRC_REVISION = readFile('version/revision.txt').trim() + } + } + } + stage('Generating version differences lists') { + steps { + bat ''' + set CURRENT_RELEASE=origin/%BRANCH_NAME% + set PREVIOUS_RELEASE=%PREVIOUS_RELEASE_BRANCH_NAME% + set WORKING_PATH=%WORKSPACE%/GitRepository + set DIFFERENCE_OPTIONS="opposite notcommon" + "%PYTHON34%" "%WORKSPACE%/print_release_difference/show_release_differences.py" > "%WORKSPACE%\\version\\differences.txt" + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Inherit version imports') { + steps { + bat ''' + call jobscripts\\version_imports.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Version stamp files') { + steps { + bat ''' + call jobscripts\\versionrcfiles.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Compile') { + steps { + bat ''' + call jobscripts\\autobuild.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Make SDK source') { + steps { + bat ''' + call jobscripts\\make_src_sdk.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Build SDK source') { + steps { + bat ''' + call jobscripts\\build_sdk.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Prepare exports') { + steps { + bat ''' + call jobscripts\\export.bat + if errorlevel 1 exit /b 1 + ''' + } + } + stage('Scan for warnings') { + steps { + warnings canComputeNew: false, canResolveRelativePaths: false, categoriesPattern: '', consoleParsers: [[parserName: 'MSBuild']], defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '', unHealthy: '' + } + } + stage('Archiving artifacts') { + steps { + archiveArtifacts 'bin/**,scripts/**,version/**,pipeline/**,sdk_source/**,sdk_source_tree_built.txt,ildasm/**' + } + } + } + post { + success { + echo 'Succeeded build' + } + } +} + From 43beb78946205895782c057a1fb3ca7cb02d05a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 11 Nov 2017 17:42:40 +0000 Subject: [PATCH 004/558] GIS-1621 - Try to parse output --- build/build.gsh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/build.gsh b/build/build.gsh index 500f20b..ffc9773 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -217,6 +217,9 @@ pipeline { success { echo 'Succeeded build' } + always { + step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) + } } } From 9814e14fb56a609bccaeaa3ee7e4b078153af419 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Nov 2017 09:54:04 +0000 Subject: [PATCH 005/558] GIS-1621 - Try to parse output --- build/build.gsh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index ffc9773..8e49430 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -214,12 +214,13 @@ pipeline { } } post { - success { + always { + echo 'Post-build' + //step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) + } + success { echo 'Succeeded build' } - always { - step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) - } - } + } } From 4b7d20c01db5e2d5af7607d5cf7535c34e30a0bd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Nov 2017 10:17:37 +0000 Subject: [PATCH 006/558] GIS-1621 - Try to parse output --- build/build.gsh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index 8e49430..499af4c 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -218,8 +218,19 @@ pipeline { echo 'Post-build' //step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) } - success { - echo 'Succeeded build' + success { + echo 'Succeeded build' + build job: '2.0.x_test', + parameters: [ + string(name: 'TEST_PORT', value: '2016'), + booleanParam(name: 'PIPELINE_NO_TEST', value: false), + booleanParam(name: 'USING_GIT', value: true), + string(name: 'REPEAT_COUNT', value: '0'), + booleanParam(name: 'QUICK_TEST', value: true), + booleanParam(name: 'LIST_TESTS', value: false), + string(name: 'BRANCH', value: 'master') + ], + propagate: false } } } From e84879a84501c56ed7c3e8acbfc5ad0efb5af4e3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Nov 2017 15:09:03 +0000 Subject: [PATCH 007/558] GIS-1621 - Try to parse output --- build/build.gsh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index 499af4c..3781d3c 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -204,7 +204,18 @@ pipeline { } stage('Scan for warnings') { steps { - warnings canComputeNew: false, canResolveRelativePaths: false, categoriesPattern: '', consoleParsers: [[parserName: 'MSBuild']], defaultEncoding: '', excludePattern: '', healthy: '', includePattern: '', messagesPattern: '', unHealthy: '' + warnings canComputeNew: false, + canResolveRelativePaths: false, + categoriesPattern: '', + consoleParsers: [ + [parserName: 'MSBuild'] + ], + defaultEncoding: '', + excludePattern: '', + healthy: '', + includePattern: '', + messagesPattern: '', + unHealthy: '' } } stage('Archiving artifacts') { @@ -216,11 +227,13 @@ pipeline { post { always { echo 'Post-build' - //step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) + } + failure { + step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) } success { echo 'Succeeded build' - build job: '2.0.x_test', + build job: '${BRANCH_NAME}_test', parameters: [ string(name: 'TEST_PORT', value: '2016'), booleanParam(name: 'PIPELINE_NO_TEST', value: false), From 60127bfce18e7379930e07fa686996b77c8c1a76 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Nov 2017 15:10:54 +0000 Subject: [PATCH 008/558] GIS-1621 - Try to parse output --- build/build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.gsh b/build/build.gsh index 3781d3c..8065b13 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -243,7 +243,7 @@ pipeline { booleanParam(name: 'LIST_TESTS', value: false), string(name: 'BRANCH', value: 'master') ], - propagate: false + propagate: true } } } From dafdaa9a08e43a9d5368fbd13faa22f4ff32c987 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Nov 2017 15:44:17 +0000 Subject: [PATCH 009/558] GIS-1621 - Expand variables properly; Format script --- build/build.gsh | 99 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index 8065b13..922899c 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -8,15 +8,28 @@ pipeline { steps { checkout([$class: 'GitSCM', branches: [ - [name: 'origin/${BRANCH_NAME}'] + [name: "origin/${BRANCH_NAME}"] ], doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: 0, noTags: false, reference: '${REPO_ROOT}/newsw', shallow: false, timeout: 60], - [$class: 'CheckoutOption', timeout: 60], - [$class: 'RelativeTargetDirectory', relativeTargetDir: 'GitRepository'], - [$class: 'PathRestriction', excludedRegions: '', includedRegions: '''src/.* autobuild/.*'''] + [$class: 'CloneOption', + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/newsw", + shallow: false, + timeout: 60 + ], + [$class: 'CheckoutOption', + timeout: 60 + ], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'GitRepository' + ], + [$class: 'PathRestriction', + excludedRegions: '', + includedRegions: '''src/.* autobuild/.*''' + ] ], submoduleCfg: [ ], @@ -30,14 +43,24 @@ pipeline { steps { checkout([$class: 'GitSCM', branches: [ - [name: 'refs/tags/${FD1200_FIRMWARE_RELEASE_VERSION}'] + [name: "refs/tags/${FD1200_FIRMWARE_RELEASE_VERSION}"] ], doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: 0, noTags: false, reference: '${REPO_ROOT}/samba_fw_releases', shallow: true, timeout: 60], - [$class: 'CheckoutOption', timeout: 60], - [$class: 'RelativeTargetDirectory', relativeTargetDir: 'samba_fw'] + [$class: 'CloneOption', + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/samba_fw_releases", + shallow: true, + timeout: 60 + ], + [$class: 'CheckoutOption', + timeout: 60 + ], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'samba_fw' + ] ], submoduleCfg: [ ], @@ -47,14 +70,24 @@ pipeline { ]) checkout([$class: 'GitSCM', branches: [ - [name: 'refs/tags/${XR_1003_EMBEDDED_LOGIC_RELEASE_VERSION}'] + [name: "refs/tags/${XR_1003_EMBEDDED_LOGIC_RELEASE_VERSION}"] ], doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: 0, noTags: false, reference: '${REPO_ROOT}/HMB_XR_1003_Embeddded_Logic.git', shallow: true, timeout: 60], - [$class: 'CheckoutOption', timeout: 60], - [$class: 'RelativeTargetDirectory', relativeTargetDir: 'xr1003_el'] + [$class: 'CloneOption', + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/HMB_XR_1003_Embeddded_Logic.git", + shallow: true, + timeout: 60 + ], + [$class: 'CheckoutOption', + timeout: 60 + ], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'xr1003_el' + ] ], submoduleCfg: [ ], @@ -64,14 +97,23 @@ pipeline { ]) checkout([$class: 'GitSCM', branches: [ - [name: 'refs/tags/${VERSION_STAMPING_VERSION}'] + [name: "refs/tags/${VERSION_STAMPING_VERSION}"] ], doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', noTags: false, reference: '${REPO_ROOT}\\ver_stamping_releases', shallow: true, timeout: 60], - [$class: 'CheckoutOption', timeout: 60], - [$class: 'RelativeTargetDirectory', relativeTargetDir: 'ver_stamping'] + [$class: 'CloneOption', + noTags: false, + reference: "${REPO_ROOT}/ver_stamping_releases", + shallow: true, + timeout: 60 + ], + [$class: 'CheckoutOption', + timeout: 60 + ], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'ver_stamping' + ] ], submoduleCfg: [], userRemoteConfigs: [ @@ -85,9 +127,18 @@ pipeline { doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', noTags: false, reference: '${REPO_ROOT}\\print_release_difference', shallow: true, timeout: 60], - [$class: 'CheckoutOption', timeout: 60], - [$class: 'RelativeTargetDirectory', relativeTargetDir: 'print_release_difference'] + [$class: 'CloneOption', + noTags: false, + reference: "${REPO_ROOT}/print_release_difference", + shallow: true, + timeout: 60 + ], + [$class: 'CheckoutOption', + timeout: 60 + ], + [$class: 'RelativeTargetDirectory', + relativeTargetDir: 'print_release_difference' + ] ], submoduleCfg: [ ], @@ -229,11 +280,14 @@ pipeline { echo 'Post-build' } failure { - step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true]) + step([$class: 'LogParserPublisher', + parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', + useProjectRule: true + ]) } success { echo 'Succeeded build' - build job: '${BRANCH_NAME}_test', + build job: "${BRANCH_NAME}_test", parameters: [ string(name: 'TEST_PORT', value: '2016'), booleanParam(name: 'PIPELINE_NO_TEST', value: false), @@ -247,4 +301,3 @@ pipeline { } } } - From b86e13a3690e13d90d451a683d1ed8ef5ca42d8d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 12 Nov 2017 16:06:51 +0000 Subject: [PATCH 010/558] GIS-1621 - Don't trigger subsequent builds; Let them trigger of this build --- build/build.gsh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index 922899c..e284106 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -280,6 +280,7 @@ pipeline { echo 'Post-build' } failure { + echo 'Failed build' step([$class: 'LogParserPublisher', parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', useProjectRule: true @@ -287,17 +288,6 @@ pipeline { } success { echo 'Succeeded build' - build job: "${BRANCH_NAME}_test", - parameters: [ - string(name: 'TEST_PORT', value: '2016'), - booleanParam(name: 'PIPELINE_NO_TEST', value: false), - booleanParam(name: 'USING_GIT', value: true), - string(name: 'REPEAT_COUNT', value: '0'), - booleanParam(name: 'QUICK_TEST', value: true), - booleanParam(name: 'LIST_TESTS', value: false), - string(name: 'BRANCH', value: 'master') - ], - propagate: true } } } From 8f30453ca6903922f740b404039a1632c59eb8e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 11:42:56 +0000 Subject: [PATCH 011/558] add README --- README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 From c62d2d3096fe786680a89f3f0d92b0689bb48515 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 16:51:58 +0000 Subject: [PATCH 012/558] Put a newline in the git included regions filter --- build/build.gsh | 141 ++++++++++++++++++++++++------------------------ 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index e284106..a398a96 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -14,22 +14,23 @@ pipeline { extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', - depth: 0, - noTags: false, - reference: "${REPO_ROOT}/newsw", - shallow: false, - timeout: 60 - ], + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/newsw", + shallow: false, + timeout: 60 + ], [$class: 'CheckoutOption', - timeout: 60 - ], + timeout: 60 + ], [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'GitRepository' - ], + relativeTargetDir: 'GitRepository' + ], [$class: 'PathRestriction', - excludedRegions: '', - includedRegions: '''src/.* autobuild/.*''' - ] + excludedRegions: '', + includedRegions: '''src/.* + autobuild/.*''' + ] ], submoduleCfg: [ ], @@ -49,18 +50,18 @@ pipeline { extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', - depth: 0, - noTags: false, - reference: "${REPO_ROOT}/samba_fw_releases", - shallow: true, - timeout: 60 - ], + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/samba_fw_releases", + shallow: true, + timeout: 60 + ], [$class: 'CheckoutOption', - timeout: 60 - ], + timeout: 60 + ], [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'samba_fw' - ] + relativeTargetDir: 'samba_fw' + ] ], submoduleCfg: [ ], @@ -76,18 +77,18 @@ pipeline { extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', - depth: 0, - noTags: false, - reference: "${REPO_ROOT}/HMB_XR_1003_Embeddded_Logic.git", - shallow: true, - timeout: 60 - ], + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/HMB_XR_1003_Embeddded_Logic.git", + shallow: true, + timeout: 60 + ], [$class: 'CheckoutOption', - timeout: 60 - ], + timeout: 60 + ], [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'xr1003_el' - ] + relativeTargetDir: 'xr1003_el' + ] ], submoduleCfg: [ ], @@ -103,17 +104,17 @@ pipeline { extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', - noTags: false, - reference: "${REPO_ROOT}/ver_stamping_releases", - shallow: true, - timeout: 60 - ], + noTags: false, + reference: "${REPO_ROOT}/ver_stamping_releases", + shallow: true, + timeout: 60 + ], [$class: 'CheckoutOption', - timeout: 60 - ], + timeout: 60 + ], [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'ver_stamping' - ] + relativeTargetDir: 'ver_stamping' + ] ], submoduleCfg: [], userRemoteConfigs: [ @@ -128,17 +129,17 @@ pipeline { extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', - noTags: false, - reference: "${REPO_ROOT}/print_release_difference", - shallow: true, - timeout: 60 - ], + noTags: false, + reference: "${REPO_ROOT}/print_release_difference", + shallow: true, + timeout: 60 + ], [$class: 'CheckoutOption', - timeout: 60 - ], + timeout: 60 + ], [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'print_release_difference' - ] + relativeTargetDir: 'print_release_difference' + ] ], submoduleCfg: [ ], @@ -256,17 +257,17 @@ pipeline { stage('Scan for warnings') { steps { warnings canComputeNew: false, - canResolveRelativePaths: false, - categoriesPattern: '', - consoleParsers: [ - [parserName: 'MSBuild'] - ], - defaultEncoding: '', - excludePattern: '', - healthy: '', - includePattern: '', - messagesPattern: '', - unHealthy: '' + canResolveRelativePaths: false, + categoriesPattern: '', + consoleParsers: [ + [parserName: 'MSBuild'] + ], + defaultEncoding: '', + excludePattern: '', + healthy: '', + includePattern: '', + messagesPattern: '', + unHealthy: '' } } stage('Archiving artifacts') { @@ -276,16 +277,16 @@ pipeline { } } post { - always { + always { echo 'Post-build' - } - failure { + } + failure { echo 'Failed build' - step([$class: 'LogParserPublisher', - parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', - useProjectRule: true - ]) - } + step([$class: 'LogParserPublisher', + parsingRulesPath: 'GitRepository/autobuild/build/log parsing rules/vs.txt', + useProjectRule: true + ]) + } success { echo 'Succeeded build' } From 3f2f2851681b8aa77c4cac08e1dd5593ec6e4ab7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 17:04:43 +0000 Subject: [PATCH 013/558] Directory filtering doesn't seem to be working - resulting in no build triggers - so removing it --- build/build.gsh | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/build/build.gsh b/build/build.gsh index a398a96..1051f40 100644 --- a/build/build.gsh +++ b/build/build.gsh @@ -14,23 +14,18 @@ pipeline { extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', - depth: 0, - noTags: false, - reference: "${REPO_ROOT}/newsw", - shallow: false, - timeout: 60 - ], + depth: 0, + noTags: false, + reference: "${REPO_ROOT}/newsw", + shallow: false, + timeout: 60 + ], [$class: 'CheckoutOption', - timeout: 60 - ], + timeout: 60 + ], [$class: 'RelativeTargetDirectory', - relativeTargetDir: 'GitRepository' - ], - [$class: 'PathRestriction', - excludedRegions: '', - includedRegions: '''src/.* - autobuild/.*''' - ] + relativeTargetDir: 'GitRepository' + ], ], submoduleCfg: [ ], From 20dd451bc8bc4f36957147469141f0c50d5fb7ed Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 20:59:01 +0000 Subject: [PATCH 014/558] Add a second test file --- README2.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 README2.md diff --git a/README2.md b/README2.md new file mode 100644 index 0000000..e69de29 From f671337ae8d333c94ee648d85d53ab91e64be610 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 21:00:04 +0000 Subject: [PATCH 015/558] Merge commit '20dd451bc8bc4f36957147469141f0c50d5fb7ed' --- README2.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 README2.md diff --git a/README2.md b/README2.md new file mode 100644 index 0000000..e69de29 From 18be423194cddea3e9f72ae71dccca2d817f316e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 21:35:34 +0000 Subject: [PATCH 016/558] Add a test groovy script --- test.gsh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 test.gsh diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..a38bfaf --- /dev/null +++ b/test.gsh @@ -0,0 +1,6 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; From a9b77d0d121b0c27ea7e50458b040df820c15fd0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 21:35:34 +0000 Subject: [PATCH 017/558] Add a test groovy script --- test.gsh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 test.gsh diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..a38bfaf --- /dev/null +++ b/test.gsh @@ -0,0 +1,6 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; From 51b4b03be8bc3eb71be393c527a34e7819766023 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 21:57:56 +0000 Subject: [PATCH 018/558] Add a test groovy script --- test.gsh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 test.gsh diff --git a/test.gsh b/test.gsh new file mode 100644 index 0000000..23fcfc0 --- /dev/null +++ b/test.gsh @@ -0,0 +1,7 @@ +// Methods in this file will end up as object methods on the object that load returns. +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; + From 61a6840f7c6de1def6be97283bd131c934af4a8b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 23:58:55 +0000 Subject: [PATCH 019/558] Start of versioning script --- versionnumbering/versioninfo.gsh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 versionnumbering/versioninfo.gsh diff --git a/versionnumbering/versioninfo.gsh b/versionnumbering/versioninfo.gsh new file mode 100644 index 0000000..579b976 --- /dev/null +++ b/versionnumbering/versioninfo.gsh @@ -0,0 +1,29 @@ +// Methods in this file will end up as object methods on the object that load returns. + +def makeVersionFolder(String versionFolder) { + dir("${versionFolder}") { + + + + + + + +def generateVersionFiles(String versionFolder, + Integer BaseGitNumber, + String SrcFolder, + String SrcBranch) { + echo """Building version folder at: ${versionFolder} + base git number: ${BaseGitNumber} + source folder: ${SrcFolder} + source branch: ${SrcBranch}""" + + dir + + + +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; From 76339b9b91f4bc895e05dd134f5234c389456a4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 28 Nov 2017 23:58:55 +0000 Subject: [PATCH 020/558] Start of versioning script --- versionnumbering/versioninfo.gsh | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 versionnumbering/versioninfo.gsh diff --git a/versionnumbering/versioninfo.gsh b/versionnumbering/versioninfo.gsh new file mode 100644 index 0000000..579b976 --- /dev/null +++ b/versionnumbering/versioninfo.gsh @@ -0,0 +1,29 @@ +// Methods in this file will end up as object methods on the object that load returns. + +def makeVersionFolder(String versionFolder) { + dir("${versionFolder}") { + + + + + + + +def generateVersionFiles(String versionFolder, + Integer BaseGitNumber, + String SrcFolder, + String SrcBranch) { + echo """Building version folder at: ${versionFolder} + base git number: ${BaseGitNumber} + source folder: ${SrcFolder} + source branch: ${SrcBranch}""" + + dir + + + +def lookAtThis(String whoAreYou) { + echo "Look at this, ${whoAreYou}! You loaded this from another file!" +} + +return this; From 9f307dd502323554587e129f7235eccfe2ea8970 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 30 Nov 2017 18:25:01 +0000 Subject: [PATCH 021/558] Add python script and eclipse project files for version numbering --- .project | 17 +++ .pydevproject | 8 ++ versionnumbering/versioninfo.py | 192 ++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 versionnumbering/versioninfo.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..d111c07 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python interpreter +python35 + diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..0567097 --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,192 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + gitvar = "GIT_EXECUTABLE" + if gitvar in os.environ and os.environ[gitvar]: + gitexecutable = os.environ[gitvar] + else: + print("Expecting GIT_EXECUTABLE environment variable to have path to git. Defaulting to 'git'") + gitexecutable = "git" + + print("Determining revision info using git command: {0}".format(gitexecutable)) + completed = subprocess.run([gitexecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitexecutable)) + completed = subprocess.run([gitexecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision) + +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0" + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return "1.20000.0." + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) From a63889509d3e7ba0961051d2618bba51840deb8f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 30 Nov 2017 18:25:01 +0000 Subject: [PATCH 022/558] Add python script and eclipse project files for version numbering --- .project | 17 +++ .pydevproject | 8 ++ versionnumbering/versioninfo.py | 192 ++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 versionnumbering/versioninfo.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..d111c07 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python interpreter +python35 + diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..0567097 --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,192 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +def resetFolder(versionfolder): + print("Removing folders and contents of: {0}".format(versionfolder)) + try: + shutil.rmtree(versionfolder) + except FileNotFoundError: + pass + + print("Creating folder: {0}".format(versionfolder)) + os.makedirs(versionfolder) + +def getGitRevision(gitrootpath, basenumber): + cwd = os.getcwd() + + path = os.path.expanduser(gitrootpath) + print("Changing current working directory to: {0}".format(path)) + os.chdir(path) + + gitvar = "GIT_EXECUTABLE" + if gitvar in os.environ and os.environ[gitvar]: + gitexecutable = os.environ[gitvar] + else: + print("Expecting GIT_EXECUTABLE environment variable to have path to git. Defaulting to 'git'") + gitexecutable = "git" + + print("Determining revision info using git command: {0}".format(gitexecutable)) + completed = subprocess.run([gitexecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + changecount = completed.stdout.decode("utf-8").strip() + print("Change count: {0}".format(changecount)) + adjustedchangecount = int(changecount) + int(basenumber) + print("Adjusted change count: {0}".format(adjustedchangecount)) + + print("Determining current revision info using git command: {0}".format(gitexecutable)) + completed = subprocess.run([gitexecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + revision = completed.stdout.decode("utf-8").strip() + print("Revision: {0}".format(revision)) + + os.chdir(cwd) + print("Restored current working directory to: {0}".format(cwd)) + + return (adjustedchangecount, revision) + +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0" + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().endswith("master"): # Special case + return "1.20000.0." + + splits = sourcebranch.split('.') + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no decimal points in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, buildnumber, buildstamp): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") + parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + + resetFolder(versionfolder) + + (gitrevisioncount, gitrevision) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) From 93caaec49afb99a61b6f324212cb24f557eb5c48 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 30 Nov 2017 18:39:49 +0000 Subject: [PATCH 023/558] Remove redundant file --- versionnumbering/versioninfo.gsh | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 versionnumbering/versioninfo.gsh diff --git a/versionnumbering/versioninfo.gsh b/versionnumbering/versioninfo.gsh deleted file mode 100644 index 579b976..0000000 --- a/versionnumbering/versioninfo.gsh +++ /dev/null @@ -1,29 +0,0 @@ -// Methods in this file will end up as object methods on the object that load returns. - -def makeVersionFolder(String versionFolder) { - dir("${versionFolder}") { - - - - - - - -def generateVersionFiles(String versionFolder, - Integer BaseGitNumber, - String SrcFolder, - String SrcBranch) { - echo """Building version folder at: ${versionFolder} - base git number: ${BaseGitNumber} - source folder: ${SrcFolder} - source branch: ${SrcBranch}""" - - dir - - - -def lookAtThis(String whoAreYou) { - echo "Look at this, ${whoAreYou}! You loaded this from another file!" -} - -return this; From 7a553a6f9e8c038fbe15120191b0969216e2aee4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 30 Nov 2017 18:39:49 +0000 Subject: [PATCH 024/558] Remove redundant file --- versionnumbering/versioninfo.gsh | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 versionnumbering/versioninfo.gsh diff --git a/versionnumbering/versioninfo.gsh b/versionnumbering/versioninfo.gsh deleted file mode 100644 index 579b976..0000000 --- a/versionnumbering/versioninfo.gsh +++ /dev/null @@ -1,29 +0,0 @@ -// Methods in this file will end up as object methods on the object that load returns. - -def makeVersionFolder(String versionFolder) { - dir("${versionFolder}") { - - - - - - - -def generateVersionFiles(String versionFolder, - Integer BaseGitNumber, - String SrcFolder, - String SrcBranch) { - echo """Building version folder at: ${versionFolder} - base git number: ${BaseGitNumber} - source folder: ${SrcFolder} - source branch: ${SrcBranch}""" - - dir - - - -def lookAtThis(String whoAreYou) { - echo "Look at this, ${whoAreYou}! You loaded this from another file!" -} - -return this; From d347fd921fc36354143f48b49434f89090d500ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 1 Dec 2017 17:03:09 +0000 Subject: [PATCH 025/558] Output git remote info into version folder --- versionnumbering/versioninfo.py | 51 ++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 0567097..35e91cd 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -59,10 +59,16 @@ def getGitRevision(gitrootpath, basenumber): revision = completed.stdout.decode("utf-8").strip() print("Revision: {0}".format(revision)) + print("Determining remote info using git command: {0}".format(gitexecutable)) + completed = subprocess.run([gitexecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Command {0} returned: {1}".format(completed.args, completed.returncode)) + remoteinfo = completed.stdout.decode("utf-8") + print("Remote info: {0}".format(remoteinfo)) + os.chdir(cwd) print("Restored current working directory to: {0}".format(cwd)) - return (adjustedchangecount, revision) + return (adjustedchangecount, revision, remoteinfo) def getVersionPrefix(sourcebranch): print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) @@ -104,7 +110,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, buildnumber, buildstamp): +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, remoteinfo, buildnumber, buildstamp): with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: f.write(buildjobname) @@ -114,6 +120,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, b with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: f.write(gitrevision) + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: f.write(buildnumber) @@ -157,31 +166,33 @@ USAGE # Process arguments args = parser.parse_args() - sourcebranch = args.sourcebranch - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber + try: + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + buildjobname = args.buildjobname + buildnumber = args.buildnumber - print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber) - resetFolder(versionfolder) + resetFolder(versionfolder) - (gitrevisioncount, gitrevision) = getGitRevision(gitrootfolder, baseversionnumber) + (gitrevisioncount, gitrevision, remoteinfo) = getGitRevision(gitrootfolder, baseversionnumber) - versionprefix = getVersionPrefix(sourcebranch) + versionprefix = getVersionPrefix(sourcebranch) - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, buildnumber, buildstamp) + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, remoteinfo, buildnumber, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 except Exception as e: indent = len(program_name) * " " print(program_name + ": " + repr(e) + "\n") From d33f6828a4e6a95abcaf4884b0d14ea9b1fad018 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Dec 2017 10:43:53 +0000 Subject: [PATCH 026/558] Add differences folder --- differences/.project | 17 +++ differences/.pydevproject | 8 ++ differences/show_release_differences.py | 151 ++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/show_release_differences.py diff --git a/differences/.project b/differences/.project new file mode 100644 index 0000000..75af0fc --- /dev/null +++ b/differences/.project @@ -0,0 +1,17 @@ + + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py new file mode 100644 index 0000000..1d525b9 --- /dev/null +++ b/differences/show_release_differences.py @@ -0,0 +1,151 @@ +''' +Created on 2 Mar 2017 + +@author: Dan +''' +import subprocess +import os +import sys +import re + +GIT_EXECUTABLE = os.getenv("GIT_EXECUTABLE") +if GIT_EXECUTABLE == None: + print("Please define the GIT_EXECUTABLE environment variable to the git executable path") + sys.exit(1) + +WORKING_PATH = os.getenv("WORKING_PATH") +if WORKING_PATH == None: + print("Please define the WORKING_PATH environment variable to the git repository path to use") + sys.exit(1) + +CURRENT_RELEASE = os.getenv("CURRENT_RELEASE") +if CURRENT_RELEASE == None: + print("Please define the CURRENT_RELEASE version") + sys.exit(1) + +PREVIOUS_RELEASE = os.getenv("PREVIOUS_RELEASE") +if PREVIOUS_RELEASE == None: + print("Please define the PREVIOUS_RELEASE version") + sys.exit(1) + +DIFFERENCE_OPTIONS = os.getenv("DIFFERENCE_OPTIONS") +if DIFFERENCE_OPTIONS == None: + DIFFERENCE_OPTIONS = "" + +class GitException(Exception): + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + +def printDivider(times = 1, length = 100): + for _ in range(times): + print("*" * length) + +TABSIZE = 4 +def printTabbed(string, tabs = 0): + print("{0}{1}".format(" " * (TABSIZE * tabs), string)) + +def getGitLogDifferences(workingPath, previousRelease, currentRelease, tripledot = False): + gitExecutable = GIT_EXECUTABLE + if tripledot: + releaseExpression = previousRelease + "..." + currentRelease + else: + releaseExpression = previousRelease + ".." + currentRelease + currentdir = os.getcwd() + os.chdir(workingPath) + gitlog = [] + printTabbed("Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if line: + linestring = line.decode('utf-8').rstrip() + gitlog.append(linestring) + #printTabbed(linestring, 1) + else: + break + proc.wait() + if proc.returncode > 0: + raise GitException("Git executable returned an error") + os.chdir(currentdir) + return gitlog + +expectedpattern = re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$") +revertpattern = re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$') +mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") + +def processLog(gitlog): + reverts = [] + merges = {} + other = [] + numbered = {} + for line in gitlog: + expectedmatch = expectedpattern.match(line) + revertmatch = revertpattern.match(line) + mergematch = mergepattern.match(line) + if expectedmatch == None and revertmatch == None and mergematch == None: + other.append(line) + if revertmatch: + gisnumber = "GIS-" + revertmatch.group(2) + description = revertmatch.group(3) + reverts.append("Revert: {0} {1}".format(gisnumber, description)) + if mergematch: + merge = mergematch.group(2) + if not merge in merges: + merges[merge] = 1 + else: + merges[merge] += 1 + if expectedmatch: + gisnumber = "GIS-" + expectedmatch.group(1) + description = expectedmatch.group(2) + if not gisnumber in numbered: + numbered[gisnumber] = [] + numbered[gisnumber].append(description) + return (reverts, merges, other, numbered) + +def showDifferences(differences): + (revertslog, mergeslog, otherlog, numberedlog) = differences + printDivider() + printTabbed("GIS-Numbers Log:") + for (gisnumber, descriptions) in sorted(numberedlog.items()): + printTabbed(gisnumber, 1) + descriptions = numberedlog[gisnumber]; + for description in descriptions: + printTabbed(description, 2) + printDivider() + printTabbed("Revert log:") + for reversion in sorted(revertslog): + printTabbed(reversion, 1) + printDivider() + printTabbed("Merges log:") + for merge in sorted(mergeslog): + printTabbed(merge, 1) + printDivider() + printTabbed("Other log:") + for other in sorted(otherlog): + printTabbed(other, 1) + +def main(): + workingPath = WORKING_PATH + previous = PREVIOUS_RELEASE + now = CURRENT_RELEASE + printDivider(times = 4) + printTabbed("Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(workingPath, previous, now)) + gitlog = getGitLogDifferences(workingPath, previous, now) + showDifferences(processLog(gitlog)) + if 'opposite' in DIFFERENCE_OPTIONS.lower(): + printDivider(times = 4) + printTabbed("Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(workingPath, now, previous)) + gitlog = getGitLogDifferences(workingPath, now, previous) + showDifferences(processLog(gitlog)) + if 'notcommon' in DIFFERENCE_OPTIONS.lower(): + printDivider(times = 4) + printTabbed("Looking for changes that are not common to {1} or {2} at path: {0}".format(workingPath, now, previous)) + gitlog = getGitLogDifferences(workingPath, now, previous, tripledot = True) + showDifferences(processLog(gitlog)) + sys.exit(0) + +if __name__ == '__main__': + main() \ No newline at end of file From e58acae19d89d804ca1bfc9b9426fe91d7f9f010 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Dec 2017 21:46:11 +0000 Subject: [PATCH 027/558] Tested version of differences generation --- .settings/org.eclipse.core.resources.prefs | 2 + differences/show_release_differences.py | 207 ++++++++++++------- version/differences.txt | 230 +++++++++++++++++++++ versionnumbering/versioninfo.py | 21 +- 4 files changed, 375 insertions(+), 85 deletions(-) create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 version/differences.txt diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 1d525b9..a778892 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -1,36 +1,26 @@ ''' -Created on 2 Mar 2017 +showreleasedifferences -- shortdesc -@author: Dan +showreleasedifferences generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. ''' import subprocess import os import sys import re -GIT_EXECUTABLE = os.getenv("GIT_EXECUTABLE") -if GIT_EXECUTABLE == None: - print("Please define the GIT_EXECUTABLE environment variable to the git executable path") - sys.exit(1) +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter -WORKING_PATH = os.getenv("WORKING_PATH") -if WORKING_PATH == None: - print("Please define the WORKING_PATH environment variable to the git repository path to use") - sys.exit(1) +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' -CURRENT_RELEASE = os.getenv("CURRENT_RELEASE") -if CURRENT_RELEASE == None: - print("Please define the CURRENT_RELEASE version") - sys.exit(1) - -PREVIOUS_RELEASE = os.getenv("PREVIOUS_RELEASE") -if PREVIOUS_RELEASE == None: - print("Please define the PREVIOUS_RELEASE version") - sys.exit(1) - -DIFFERENCE_OPTIONS = os.getenv("DIFFERENCE_OPTIONS") -if DIFFERENCE_OPTIONS == None: - DIFFERENCE_OPTIONS = "" +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") class GitException(Exception): def __init__(self, message): @@ -39,31 +29,34 @@ class GitException(Exception): def __repr__(self): return self.message -def printDivider(times = 1, length = 100): +def makeFolder(versionfolder): + print("Creating folder (if not already there): {0}".format(versionfolder)) + os.makedirs(versionfolder, exist_ok = True) + +def printDivider(outfh, times = 1, length = 100): for _ in range(times): - print("*" * length) + outfh.write("{0}\n".format("*" * length)) TABSIZE = 4 -def printTabbed(string, tabs = 0): - print("{0}{1}".format(" " * (TABSIZE * tabs), string)) +def printTabbed(outfh, string, tabs = 0): + outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) -def getGitLogDifferences(workingPath, previousRelease, currentRelease, tripledot = False): - gitExecutable = GIT_EXECUTABLE +def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): if tripledot: releaseExpression = previousRelease + "..." + currentRelease else: releaseExpression = previousRelease + ".." + currentRelease currentdir = os.getcwd() - os.chdir(workingPath) + os.chdir(gitrootfolder) gitlog = [] - printTabbed("Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) while True: line = proc.stdout.readline() if line: linestring = line.decode('utf-8').rstrip() gitlog.append(linestring) - #printTabbed(linestring, 1) + #printTabbed(outfh, linestring, 1) else: break proc.wait() @@ -72,15 +65,23 @@ def getGitLogDifferences(workingPath, previousRelease, currentRelease, tripledot os.chdir(currentdir) return gitlog -expectedpattern = re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$") -revertpattern = re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$') +jiraprojectpattern = { + 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), + ), + 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), + ), + } + mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") -def processLog(gitlog): +def processLog(gitlog, jirakeyprefix): reverts = [] merges = {} other = [] numbered = {} + (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] for line in gitlog: expectedmatch = expectedpattern.match(line) revertmatch = revertpattern.match(line) @@ -88,7 +89,7 @@ def processLog(gitlog): if expectedmatch == None and revertmatch == None and mergematch == None: other.append(line) if revertmatch: - gisnumber = "GIS-" + revertmatch.group(2) + gisnumber = jirakeyprefix + "-" + revertmatch.group(2) description = revertmatch.group(3) reverts.append("Revert: {0} {1}".format(gisnumber, description)) if mergematch: @@ -98,54 +99,116 @@ def processLog(gitlog): else: merges[merge] += 1 if expectedmatch: - gisnumber = "GIS-" + expectedmatch.group(1) + gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) description = expectedmatch.group(2) if not gisnumber in numbered: numbered[gisnumber] = [] numbered[gisnumber].append(description) return (reverts, merges, other, numbered) -def showDifferences(differences): +def showDifferences(differences, jirakeyprefix, outfh): (revertslog, mergeslog, otherlog, numberedlog) = differences - printDivider() - printTabbed("GIS-Numbers Log:") + printDivider(outfh) + printTabbed(outfh, jirakeyprefix + "-Numbers Log:") for (gisnumber, descriptions) in sorted(numberedlog.items()): - printTabbed(gisnumber, 1) + printTabbed(outfh, gisnumber, 1) descriptions = numberedlog[gisnumber]; for description in descriptions: - printTabbed(description, 2) - printDivider() - printTabbed("Revert log:") + printTabbed(outfh, description, 2) + printDivider(outfh) + printTabbed(outfh, "Revert log:") for reversion in sorted(revertslog): - printTabbed(reversion, 1) - printDivider() - printTabbed("Merges log:") + printTabbed(outfh, reversion, 1) + printDivider(outfh) + printTabbed(outfh, "Merges log:") for merge in sorted(mergeslog): - printTabbed(merge, 1) - printDivider() - printTabbed("Other log:") + printTabbed(outfh, merge, 1) + printDivider(outfh) + printTabbed(outfh, "Other log:") for other in sorted(otherlog): - printTabbed(other, 1) + printTabbed(outfh, other, 1) -def main(): - workingPath = WORKING_PATH - previous = PREVIOUS_RELEASE - now = CURRENT_RELEASE - printDivider(times = 4) - printTabbed("Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(workingPath, previous, now)) - gitlog = getGitLogDifferences(workingPath, previous, now) - showDifferences(processLog(gitlog)) - if 'opposite' in DIFFERENCE_OPTIONS.lower(): - printDivider(times = 4) - printTabbed("Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(workingPath, now, previous)) - gitlog = getGitLogDifferences(workingPath, now, previous) - showDifferences(processLog(gitlog)) - if 'notcommon' in DIFFERENCE_OPTIONS.lower(): - printDivider(times = 4) - printTabbed("Looking for changes that are not common to {1} or {2} at path: {0}".format(workingPath, now, previous)) - gitlog = getGitLogDifferences(workingPath, now, previous, tripledot = True) - showDifferences(processLog(gitlog)) - sys.exit(0) +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' -if __name__ == '__main__': - main() \ No newline at end of file + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") + parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") + parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") + parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + gitrootfolder = args.gitrootfolder + sourceversion = args.sourceversion + previousversion = args.previousversion + jirakeyprefix = args.jirakeyprefix + outputoptions = args.outputoptions + versionfolder = args.versionfolder + + print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + + if versionfolder: + makeFolder(versionfolder) + outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + else: + outfh = sys.stdout + + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'opposite' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'notcommon' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + + if versionfolder: + outfh.close() + return 0 + except Exception as e: + print("Error while determining differences:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/version/differences.txt b/version/differences.txt new file mode 100644 index 0000000..7ec3ad6 --- /dev/null +++ b/version/differences.txt @@ -0,0 +1,230 @@ +**************************************************************************************************** +**************************************************************************************************** +**************************************************************************************************** +**************************************************************************************************** +Looking for changes in origin/master that are not in origin/1.5.4 of the repository at path: /home/dan/home2/Development/inksystem +Executing: git log --oneline origin/1.5.4..origin/master +**************************************************************************************************** +IMB-Numbers Log: + IMB-11 + Further minor change to tooltip + Changed tooltip for PWN Frequency on ink pumps + Changed the Tooltips for PWM frequency and duty cycle + IMB-12 + Changed the order the events are generated in to prevent master from resetting slave + When a vent command is sent to a slave, it will demand a vent from its master + IMB-14 + Fixed state transition in Controlled Flow Master where it could get stuck in pre-flow + IMB-4 + - Add more logging + IMB-42 + Master sets the slaves to dormant when pre-idling + IMB-43 + Added purge abort for HP Type 2 + Fixed shared controlled flow transition for purge all slaves + Added Idle transitions from Purge + Added missing definition + Added explicit transitions for standby for all states for flowmode 1-3 + IMB-47 + - Tidied up checkStateR to be consistent with setValueR + - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. + - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. + IMB-50 + Temperature setpoint set when entering flow configuration to ensure correct heater operation + IMB-6 + Removed a white-space in the output + IMB-7 + Changed the formatting of the output + IMB-8 + When returning from a purge it now calls the same code as when going into idle, as they effectively were identical. + Added pre-flow to post-flow transitions and vice-versa + In gravity mode: removed the activation of the vacuum pump which speeds up ramps. This caused vent and pump to "fight" each other +**************************************************************************************************** +Revert log: +**************************************************************************************************** +Merges log: + Merge branch '1.10023.j6221-Type2_Pneumatic' of hydra:gis/inksystem into 1.10023.j6221-Type2_Pneumatic + Merge branch 'master' of hydra:gis/inksystem + Merge remote-tracking branch 'remotes/origin/1.10025.j6416-EthernetCongestion' into 1.10023.j6221-Type2_Pneumatic + Merge remote-tracking branch 'remotes/origin/1.5.2' into 1.10023.j6221-Type2_Pneumatic + Merge remote-tracking branch 'remotes/origin/1.5.5' + Merge remote-tracking branch 'remotes/origin/master' into 1.10023.j6221-Type2_Pneumatic +**************************************************************************************************** +Other log: + 095290c GIS-1621 - Make compilation work in case-sensitive linux + 0a8505d GIS-1621 - Remove test code + 0e8ccce GIS-6380 Removed ambiguous "absolute pressure" terminology + 149e0bf GIS 6221: Added logging for gravity mode + 169125b GIS-6221: Corrected Typo + 23c37d0 GIS 43: System will abort purge in a safe manner if standby command is issued + 37dd24b GIS 6479: Controlled flow master now goes through pre-flow. Fixed ordering in controlevent.c and .h + 41ac373 GIS-6253: Fixed bugs + 45cdfbe GIS 6591: Fixed if statement which outputs ITPD and OTPD + 468798e Update release notes whitespace + 4c2badc GIS-1621 - 16 bit integer is not printing correctly + 4c9a1aa GIS-6221 Vent the outlet bottle when purging to prevent "kick" on return from idle + 541c3bb GIS-6253: Added gravity flowmode + 5562461 GIS-6570 - Handle 0 length buffers being sent for TCP transmission; Add some logging for diagnosis + 577444e Update release notes for 1.5.5 + 5c6416b GIS-6416 - Instrument number of buffers on socket transmit queue and grapher empty buffer queue + 603de5a GIS-6221: Fixed more typos + 628b57d GIS 6221: Fixed typo + 6526df4 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. + 6da6243 GIS-6221 Changed purge configuration, return from purge still to do + 7d263d0 GIS 6591: Change the output of the settings command on Telnet for clarity + 83f5517 Update 1.5.5 release notes with known issues + 85fc901 GIS-6570 - Handle condition when no bytes are consumed by sck_send (because the connection has died) for TCP transmission; Add some logging for diagnosis + 8a2db28 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. + 8df4c1b GIS-1621 - Settings BSP files have changed unexpectedly from 1.5.4 - maybe causing a slow down + 92c135e GIS 6462 Fixed typo + 9400923 GIS-6516: Slave resets from vent when master is reset + 9964521 GIS 6253: Added methods for gravity mode logging in inkchannelinterface.cpp/h + a1f0025 GIS 43: Added PreIdle with slaves in controlled flow master + a900a24 GIS-6221: Changed Idle to be driven from Inlet Side + b2f4383 GIS-1620 - Correct string print + b42ea46 GIS-1621 - Handle version prefix + b498cd5 GIS 6462 Added on the fly idle pressure change in shared pressure controlled flow + b4e1b8c GIS 6221: Resolved merge conflicts and applied fixes from 1.5.3 to the Type 2 High Pressure mode + bbc5e45 GIS 6221 - Changed High Pressure Type 2 to flowmode 6 + bd69b60 GIS-6253: Fixed typos + c568789 GIS 6221 - Bug fixes High Pressure Type 2 Flow mode + c666970 GIS-6570 - Add some logging for diagnosis + c7bbf29 Fixed shared controlled flow transition for purge all slaves to standby + c962c65 GIS-6221 Added Purge Return via Idle + ca1e7b5 GIS-6253: Added the files for gravity flowmode + ca4473b GIS-1621 - Only send pressure updates if there is a 4 point difference in value + cb06440 GIS 6221: Changed descriptions of Flowmodes. Changed output of settings to show purge pressure for flowmode 5 and 6 + d4dd54d GIS 6462: Added on-the-fly pressure change for Shared Pressure Controlled Flow systems + da69186 GIS 6492: Dumpconfig now outputs the software version + df64abd Revert "GIS-1621 - Only send pressure updates if there is a 4 point difference in value" + e51971f GIS 6379: The purge pressure is only shown on High Pressure systems (flowmode 4 and 5) + e6b0227 GIS-6221 Corrected vent operation for outlet bottle + ec13b60 GIS-6416 - Handle multiple writes to a TCP stream until it becomes busy + f018a86 GIS 6479: Added an applysettings command to allow on the fly pressure and temperature changes + f7109ff GIS 43: Added purge to idle transitions for Gravity Mode + f87280b GIS 43: Fixed transition in cfm + f999765 GIS-6253: Removed redundant function + ff37df8 GIS-6253: Fixed typos + ff92b79 GIS 6591: Added statement on Master settings +**************************************************************************************************** +**************************************************************************************************** +**************************************************************************************************** +**************************************************************************************************** +Looking for changes in origin/1.5.4 that are not in origin/master of the repository at path: /home/dan/home2/Development/inksystem +Executing: git log --oneline origin/master..origin/1.5.4 +**************************************************************************************************** +IMB-Numbers Log: +**************************************************************************************************** +Revert log: +**************************************************************************************************** +Merges log: +**************************************************************************************************** +Other log: +**************************************************************************************************** +**************************************************************************************************** +**************************************************************************************************** +**************************************************************************************************** +Looking for changes that are not common to origin/master or origin/1.5.4 at path: /home/dan/home2/Development/inksystem +Executing: git log --oneline origin/master...origin/1.5.4 +**************************************************************************************************** +IMB-Numbers Log: + IMB-11 + Further minor change to tooltip + Changed tooltip for PWN Frequency on ink pumps + Changed the Tooltips for PWM frequency and duty cycle + IMB-12 + Changed the order the events are generated in to prevent master from resetting slave + When a vent command is sent to a slave, it will demand a vent from its master + IMB-14 + Fixed state transition in Controlled Flow Master where it could get stuck in pre-flow + IMB-4 + - Add more logging + IMB-42 + Master sets the slaves to dormant when pre-idling + IMB-43 + Added purge abort for HP Type 2 + Fixed shared controlled flow transition for purge all slaves + Added Idle transitions from Purge + Added missing definition + Added explicit transitions for standby for all states for flowmode 1-3 + IMB-47 + - Tidied up checkStateR to be consistent with setValueR + - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. + - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. + IMB-50 + Temperature setpoint set when entering flow configuration to ensure correct heater operation + IMB-6 + Removed a white-space in the output + IMB-7 + Changed the formatting of the output + IMB-8 + When returning from a purge it now calls the same code as when going into idle, as they effectively were identical. + Added pre-flow to post-flow transitions and vice-versa + In gravity mode: removed the activation of the vacuum pump which speeds up ramps. This caused vent and pump to "fight" each other +**************************************************************************************************** +Revert log: +**************************************************************************************************** +Merges log: + Merge branch '1.10023.j6221-Type2_Pneumatic' of hydra:gis/inksystem into 1.10023.j6221-Type2_Pneumatic + Merge branch 'master' of hydra:gis/inksystem + Merge remote-tracking branch 'remotes/origin/1.10025.j6416-EthernetCongestion' into 1.10023.j6221-Type2_Pneumatic + Merge remote-tracking branch 'remotes/origin/1.5.2' into 1.10023.j6221-Type2_Pneumatic + Merge remote-tracking branch 'remotes/origin/1.5.5' + Merge remote-tracking branch 'remotes/origin/master' into 1.10023.j6221-Type2_Pneumatic +**************************************************************************************************** +Other log: + 095290c GIS-1621 - Make compilation work in case-sensitive linux + 0a8505d GIS-1621 - Remove test code + 0e8ccce GIS-6380 Removed ambiguous "absolute pressure" terminology + 149e0bf GIS 6221: Added logging for gravity mode + 169125b GIS-6221: Corrected Typo + 23c37d0 GIS 43: System will abort purge in a safe manner if standby command is issued + 37dd24b GIS 6479: Controlled flow master now goes through pre-flow. Fixed ordering in controlevent.c and .h + 41ac373 GIS-6253: Fixed bugs + 45cdfbe GIS 6591: Fixed if statement which outputs ITPD and OTPD + 468798e Update release notes whitespace + 4c2badc GIS-1621 - 16 bit integer is not printing correctly + 4c9a1aa GIS-6221 Vent the outlet bottle when purging to prevent "kick" on return from idle + 541c3bb GIS-6253: Added gravity flowmode + 5562461 GIS-6570 - Handle 0 length buffers being sent for TCP transmission; Add some logging for diagnosis + 577444e Update release notes for 1.5.5 + 5c6416b GIS-6416 - Instrument number of buffers on socket transmit queue and grapher empty buffer queue + 603de5a GIS-6221: Fixed more typos + 628b57d GIS 6221: Fixed typo + 6526df4 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. + 6da6243 GIS-6221 Changed purge configuration, return from purge still to do + 7d263d0 GIS 6591: Change the output of the settings command on Telnet for clarity + 83f5517 Update 1.5.5 release notes with known issues + 85fc901 GIS-6570 - Handle condition when no bytes are consumed by sck_send (because the connection has died) for TCP transmission; Add some logging for diagnosis + 8a2db28 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. + 8df4c1b GIS-1621 - Settings BSP files have changed unexpectedly from 1.5.4 - maybe causing a slow down + 92c135e GIS 6462 Fixed typo + 9400923 GIS-6516: Slave resets from vent when master is reset + 9964521 GIS 6253: Added methods for gravity mode logging in inkchannelinterface.cpp/h + a1f0025 GIS 43: Added PreIdle with slaves in controlled flow master + a900a24 GIS-6221: Changed Idle to be driven from Inlet Side + b2f4383 GIS-1620 - Correct string print + b42ea46 GIS-1621 - Handle version prefix + b498cd5 GIS 6462 Added on the fly idle pressure change in shared pressure controlled flow + b4e1b8c GIS 6221: Resolved merge conflicts and applied fixes from 1.5.3 to the Type 2 High Pressure mode + bbc5e45 GIS 6221 - Changed High Pressure Type 2 to flowmode 6 + bd69b60 GIS-6253: Fixed typos + c568789 GIS 6221 - Bug fixes High Pressure Type 2 Flow mode + c666970 GIS-6570 - Add some logging for diagnosis + c7bbf29 Fixed shared controlled flow transition for purge all slaves to standby + c962c65 GIS-6221 Added Purge Return via Idle + ca1e7b5 GIS-6253: Added the files for gravity flowmode + ca4473b GIS-1621 - Only send pressure updates if there is a 4 point difference in value + cb06440 GIS 6221: Changed descriptions of Flowmodes. Changed output of settings to show purge pressure for flowmode 5 and 6 + d4dd54d GIS 6462: Added on-the-fly pressure change for Shared Pressure Controlled Flow systems + da69186 GIS 6492: Dumpconfig now outputs the software version + df64abd Revert "GIS-1621 - Only send pressure updates if there is a 4 point difference in value" + e51971f GIS 6379: The purge pressure is only shown on High Pressure systems (flowmode 4 and 5) + e6b0227 GIS-6221 Corrected vent operation for outlet bottle + ec13b60 GIS-6416 - Handle multiple writes to a TCP stream until it becomes busy + f018a86 GIS 6479: Added an applysettings command to allow on the fly pressure and temperature changes + f7109ff GIS 43: Added purge to idle transitions for Gravity Mode + f87280b GIS 43: Fixed transition in cfm + f999765 GIS-6253: Removed redundant function + ff37df8 GIS-6253: Fixed typos + ff92b79 GIS 6591: Added statement on Master settings diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 35e91cd..c8c79fe 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -21,6 +21,8 @@ __version__ = 0.1 __date__ = '2017-11-29' __updated__ = '2017-11-29' +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") + def resetFolder(versionfolder): print("Removing folders and contents of: {0}".format(versionfolder)) try: @@ -38,29 +40,22 @@ def getGitRevision(gitrootpath, basenumber): print("Changing current working directory to: {0}".format(path)) os.chdir(path) - gitvar = "GIT_EXECUTABLE" - if gitvar in os.environ and os.environ[gitvar]: - gitexecutable = os.environ[gitvar] - else: - print("Expecting GIT_EXECUTABLE environment variable to have path to git. Defaulting to 'git'") - gitexecutable = "git" - - print("Determining revision info using git command: {0}".format(gitexecutable)) - completed = subprocess.run([gitexecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) + print("Determining revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) adjustedchangecount = int(changecount) + int(basenumber) print("Adjusted change count: {0}".format(adjustedchangecount)) - print("Determining current revision info using git command: {0}".format(gitexecutable)) - completed = subprocess.run([gitexecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) + print("Determining current revision info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) print("Command {0} returned: {1}".format(completed.args, completed.returncode)) revision = completed.stdout.decode("utf-8").strip() print("Revision: {0}".format(revision)) - print("Determining remote info using git command: {0}".format(gitexecutable)) - completed = subprocess.run([gitexecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) + print("Determining remote info using git command: {0}".format(gitExecutable)) + completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) print("Command {0} returned: {1}".format(completed.args, completed.returncode)) remoteinfo = completed.stdout.decode("utf-8") print("Remote info: {0}".format(remoteinfo)) From 2fab1a566ec16460835b4aefda492ad56b435490 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 2 Dec 2017 21:48:37 +0000 Subject: [PATCH 028/558] Remove redundant files --- README.md | 0 README2.md | 0 version/differences.txt | 230 ---------------------------------------- 3 files changed, 230 deletions(-) delete mode 100644 README.md delete mode 100644 README2.md delete mode 100644 version/differences.txt diff --git a/README.md b/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/README2.md b/README2.md deleted file mode 100644 index e69de29..0000000 diff --git a/version/differences.txt b/version/differences.txt deleted file mode 100644 index 7ec3ad6..0000000 --- a/version/differences.txt +++ /dev/null @@ -1,230 +0,0 @@ -**************************************************************************************************** -**************************************************************************************************** -**************************************************************************************************** -**************************************************************************************************** -Looking for changes in origin/master that are not in origin/1.5.4 of the repository at path: /home/dan/home2/Development/inksystem -Executing: git log --oneline origin/1.5.4..origin/master -**************************************************************************************************** -IMB-Numbers Log: - IMB-11 - Further minor change to tooltip - Changed tooltip for PWN Frequency on ink pumps - Changed the Tooltips for PWM frequency and duty cycle - IMB-12 - Changed the order the events are generated in to prevent master from resetting slave - When a vent command is sent to a slave, it will demand a vent from its master - IMB-14 - Fixed state transition in Controlled Flow Master where it could get stuck in pre-flow - IMB-4 - - Add more logging - IMB-42 - Master sets the slaves to dormant when pre-idling - IMB-43 - Added purge abort for HP Type 2 - Fixed shared controlled flow transition for purge all slaves - Added Idle transitions from Purge - Added missing definition - Added explicit transitions for standby for all states for flowmode 1-3 - IMB-47 - - Tidied up checkStateR to be consistent with setValueR - - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. - - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. - IMB-50 - Temperature setpoint set when entering flow configuration to ensure correct heater operation - IMB-6 - Removed a white-space in the output - IMB-7 - Changed the formatting of the output - IMB-8 - When returning from a purge it now calls the same code as when going into idle, as they effectively were identical. - Added pre-flow to post-flow transitions and vice-versa - In gravity mode: removed the activation of the vacuum pump which speeds up ramps. This caused vent and pump to "fight" each other -**************************************************************************************************** -Revert log: -**************************************************************************************************** -Merges log: - Merge branch '1.10023.j6221-Type2_Pneumatic' of hydra:gis/inksystem into 1.10023.j6221-Type2_Pneumatic - Merge branch 'master' of hydra:gis/inksystem - Merge remote-tracking branch 'remotes/origin/1.10025.j6416-EthernetCongestion' into 1.10023.j6221-Type2_Pneumatic - Merge remote-tracking branch 'remotes/origin/1.5.2' into 1.10023.j6221-Type2_Pneumatic - Merge remote-tracking branch 'remotes/origin/1.5.5' - Merge remote-tracking branch 'remotes/origin/master' into 1.10023.j6221-Type2_Pneumatic -**************************************************************************************************** -Other log: - 095290c GIS-1621 - Make compilation work in case-sensitive linux - 0a8505d GIS-1621 - Remove test code - 0e8ccce GIS-6380 Removed ambiguous "absolute pressure" terminology - 149e0bf GIS 6221: Added logging for gravity mode - 169125b GIS-6221: Corrected Typo - 23c37d0 GIS 43: System will abort purge in a safe manner if standby command is issued - 37dd24b GIS 6479: Controlled flow master now goes through pre-flow. Fixed ordering in controlevent.c and .h - 41ac373 GIS-6253: Fixed bugs - 45cdfbe GIS 6591: Fixed if statement which outputs ITPD and OTPD - 468798e Update release notes whitespace - 4c2badc GIS-1621 - 16 bit integer is not printing correctly - 4c9a1aa GIS-6221 Vent the outlet bottle when purging to prevent "kick" on return from idle - 541c3bb GIS-6253: Added gravity flowmode - 5562461 GIS-6570 - Handle 0 length buffers being sent for TCP transmission; Add some logging for diagnosis - 577444e Update release notes for 1.5.5 - 5c6416b GIS-6416 - Instrument number of buffers on socket transmit queue and grapher empty buffer queue - 603de5a GIS-6221: Fixed more typos - 628b57d GIS 6221: Fixed typo - 6526df4 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. - 6da6243 GIS-6221 Changed purge configuration, return from purge still to do - 7d263d0 GIS 6591: Change the output of the settings command on Telnet for clarity - 83f5517 Update 1.5.5 release notes with known issues - 85fc901 GIS-6570 - Handle condition when no bytes are consumed by sck_send (because the connection has died) for TCP transmission; Add some logging for diagnosis - 8a2db28 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. - 8df4c1b GIS-1621 - Settings BSP files have changed unexpectedly from 1.5.4 - maybe causing a slow down - 92c135e GIS 6462 Fixed typo - 9400923 GIS-6516: Slave resets from vent when master is reset - 9964521 GIS 6253: Added methods for gravity mode logging in inkchannelinterface.cpp/h - a1f0025 GIS 43: Added PreIdle with slaves in controlled flow master - a900a24 GIS-6221: Changed Idle to be driven from Inlet Side - b2f4383 GIS-1620 - Correct string print - b42ea46 GIS-1621 - Handle version prefix - b498cd5 GIS 6462 Added on the fly idle pressure change in shared pressure controlled flow - b4e1b8c GIS 6221: Resolved merge conflicts and applied fixes from 1.5.3 to the Type 2 High Pressure mode - bbc5e45 GIS 6221 - Changed High Pressure Type 2 to flowmode 6 - bd69b60 GIS-6253: Fixed typos - c568789 GIS 6221 - Bug fixes High Pressure Type 2 Flow mode - c666970 GIS-6570 - Add some logging for diagnosis - c7bbf29 Fixed shared controlled flow transition for purge all slaves to standby - c962c65 GIS-6221 Added Purge Return via Idle - ca1e7b5 GIS-6253: Added the files for gravity flowmode - ca4473b GIS-1621 - Only send pressure updates if there is a 4 point difference in value - cb06440 GIS 6221: Changed descriptions of Flowmodes. Changed output of settings to show purge pressure for flowmode 5 and 6 - d4dd54d GIS 6462: Added on-the-fly pressure change for Shared Pressure Controlled Flow systems - da69186 GIS 6492: Dumpconfig now outputs the software version - df64abd Revert "GIS-1621 - Only send pressure updates if there is a 4 point difference in value" - e51971f GIS 6379: The purge pressure is only shown on High Pressure systems (flowmode 4 and 5) - e6b0227 GIS-6221 Corrected vent operation for outlet bottle - ec13b60 GIS-6416 - Handle multiple writes to a TCP stream until it becomes busy - f018a86 GIS 6479: Added an applysettings command to allow on the fly pressure and temperature changes - f7109ff GIS 43: Added purge to idle transitions for Gravity Mode - f87280b GIS 43: Fixed transition in cfm - f999765 GIS-6253: Removed redundant function - ff37df8 GIS-6253: Fixed typos - ff92b79 GIS 6591: Added statement on Master settings -**************************************************************************************************** -**************************************************************************************************** -**************************************************************************************************** -**************************************************************************************************** -Looking for changes in origin/1.5.4 that are not in origin/master of the repository at path: /home/dan/home2/Development/inksystem -Executing: git log --oneline origin/master..origin/1.5.4 -**************************************************************************************************** -IMB-Numbers Log: -**************************************************************************************************** -Revert log: -**************************************************************************************************** -Merges log: -**************************************************************************************************** -Other log: -**************************************************************************************************** -**************************************************************************************************** -**************************************************************************************************** -**************************************************************************************************** -Looking for changes that are not common to origin/master or origin/1.5.4 at path: /home/dan/home2/Development/inksystem -Executing: git log --oneline origin/master...origin/1.5.4 -**************************************************************************************************** -IMB-Numbers Log: - IMB-11 - Further minor change to tooltip - Changed tooltip for PWN Frequency on ink pumps - Changed the Tooltips for PWM frequency and duty cycle - IMB-12 - Changed the order the events are generated in to prevent master from resetting slave - When a vent command is sent to a slave, it will demand a vent from its master - IMB-14 - Fixed state transition in Controlled Flow Master where it could get stuck in pre-flow - IMB-4 - - Add more logging - IMB-42 - Master sets the slaves to dormant when pre-idling - IMB-43 - Added purge abort for HP Type 2 - Fixed shared controlled flow transition for purge all slaves - Added Idle transitions from Purge - Added missing definition - Added explicit transitions for standby for all states for flowmode 1-3 - IMB-47 - - Tidied up checkStateR to be consistent with setValueR - - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. - - The thermistor will only generate an event if a threshold has been crossed, or when "timeToSendAnotherEvent" is true. - IMB-50 - Temperature setpoint set when entering flow configuration to ensure correct heater operation - IMB-6 - Removed a white-space in the output - IMB-7 - Changed the formatting of the output - IMB-8 - When returning from a purge it now calls the same code as when going into idle, as they effectively were identical. - Added pre-flow to post-flow transitions and vice-versa - In gravity mode: removed the activation of the vacuum pump which speeds up ramps. This caused vent and pump to "fight" each other -**************************************************************************************************** -Revert log: -**************************************************************************************************** -Merges log: - Merge branch '1.10023.j6221-Type2_Pneumatic' of hydra:gis/inksystem into 1.10023.j6221-Type2_Pneumatic - Merge branch 'master' of hydra:gis/inksystem - Merge remote-tracking branch 'remotes/origin/1.10025.j6416-EthernetCongestion' into 1.10023.j6221-Type2_Pneumatic - Merge remote-tracking branch 'remotes/origin/1.5.2' into 1.10023.j6221-Type2_Pneumatic - Merge remote-tracking branch 'remotes/origin/1.5.5' - Merge remote-tracking branch 'remotes/origin/master' into 1.10023.j6221-Type2_Pneumatic -**************************************************************************************************** -Other log: - 095290c GIS-1621 - Make compilation work in case-sensitive linux - 0a8505d GIS-1621 - Remove test code - 0e8ccce GIS-6380 Removed ambiguous "absolute pressure" terminology - 149e0bf GIS 6221: Added logging for gravity mode - 169125b GIS-6221: Corrected Typo - 23c37d0 GIS 43: System will abort purge in a safe manner if standby command is issued - 37dd24b GIS 6479: Controlled flow master now goes through pre-flow. Fixed ordering in controlevent.c and .h - 41ac373 GIS-6253: Fixed bugs - 45cdfbe GIS 6591: Fixed if statement which outputs ITPD and OTPD - 468798e Update release notes whitespace - 4c2badc GIS-1621 - 16 bit integer is not printing correctly - 4c9a1aa GIS-6221 Vent the outlet bottle when purging to prevent "kick" on return from idle - 541c3bb GIS-6253: Added gravity flowmode - 5562461 GIS-6570 - Handle 0 length buffers being sent for TCP transmission; Add some logging for diagnosis - 577444e Update release notes for 1.5.5 - 5c6416b GIS-6416 - Instrument number of buffers on socket transmit queue and grapher empty buffer queue - 603de5a GIS-6221: Fixed more typos - 628b57d GIS 6221: Fixed typo - 6526df4 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. - 6da6243 GIS-6221 Changed purge configuration, return from purge still to do - 7d263d0 GIS 6591: Change the output of the settings command on Telnet for clarity - 83f5517 Update 1.5.5 release notes with known issues - 85fc901 GIS-6570 - Handle condition when no bytes are consumed by sck_send (because the connection has died) for TCP transmission; Add some logging for diagnosis - 8a2db28 GIS-6221 Corrected behaviour at idle. When idling, the pressure is now only vacuum pump driven in one direction. - 8df4c1b GIS-1621 - Settings BSP files have changed unexpectedly from 1.5.4 - maybe causing a slow down - 92c135e GIS 6462 Fixed typo - 9400923 GIS-6516: Slave resets from vent when master is reset - 9964521 GIS 6253: Added methods for gravity mode logging in inkchannelinterface.cpp/h - a1f0025 GIS 43: Added PreIdle with slaves in controlled flow master - a900a24 GIS-6221: Changed Idle to be driven from Inlet Side - b2f4383 GIS-1620 - Correct string print - b42ea46 GIS-1621 - Handle version prefix - b498cd5 GIS 6462 Added on the fly idle pressure change in shared pressure controlled flow - b4e1b8c GIS 6221: Resolved merge conflicts and applied fixes from 1.5.3 to the Type 2 High Pressure mode - bbc5e45 GIS 6221 - Changed High Pressure Type 2 to flowmode 6 - bd69b60 GIS-6253: Fixed typos - c568789 GIS 6221 - Bug fixes High Pressure Type 2 Flow mode - c666970 GIS-6570 - Add some logging for diagnosis - c7bbf29 Fixed shared controlled flow transition for purge all slaves to standby - c962c65 GIS-6221 Added Purge Return via Idle - ca1e7b5 GIS-6253: Added the files for gravity flowmode - ca4473b GIS-1621 - Only send pressure updates if there is a 4 point difference in value - cb06440 GIS 6221: Changed descriptions of Flowmodes. Changed output of settings to show purge pressure for flowmode 5 and 6 - d4dd54d GIS 6462: Added on-the-fly pressure change for Shared Pressure Controlled Flow systems - da69186 GIS 6492: Dumpconfig now outputs the software version - df64abd Revert "GIS-1621 - Only send pressure updates if there is a 4 point difference in value" - e51971f GIS 6379: The purge pressure is only shown on High Pressure systems (flowmode 4 and 5) - e6b0227 GIS-6221 Corrected vent operation for outlet bottle - ec13b60 GIS-6416 - Handle multiple writes to a TCP stream until it becomes busy - f018a86 GIS 6479: Added an applysettings command to allow on the fly pressure and temperature changes - f7109ff GIS 43: Added purge to idle transitions for Gravity Mode - f87280b GIS 43: Fixed transition in cfm - f999765 GIS-6253: Removed redundant function - ff37df8 GIS-6253: Fixed typos - ff92b79 GIS 6591: Added statement on Master settings From 4dbbbf2b4247eb39770d2c9f4d4498ed1f4a8c0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Dec 2017 13:13:23 +0000 Subject: [PATCH 029/558] Provide a comma seperated build stamp --- versionnumbering/versioninfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index c8c79fe..a4162e0 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -124,6 +124,10 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, r with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: f.write(buildstamp) + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + def main(argv=None): # IGNORE:C0111 '''Command line options.''' From 584a016c493000ae456182f37b4c50bdf739f9c9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Dec 2017 13:33:27 +0000 Subject: [PATCH 030/558] Move code to common position --- build/eclipsebuild/eclipsebuild.gsh | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 build/eclipsebuild/eclipsebuild.gsh diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh new file mode 100644 index 0000000..e7dcd09 --- /dev/null +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -0,0 +1,34 @@ +// Build and export files +def doBuild(apps) { + for (appInfo in apps ) { + if (appInfo[0]) + { + retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) + if (isUnix()) { + sh "${BUILD_COMMAND} -build ${appInfo[1]}/${BUILD_VARIANT}" + } + else { + bat "%BUILD_COMMAND% -build ${appInfo[1]}/${BUILD_VARIANT}" + } + } + echo "Looking at: ${appInfo[2]}" + script { + env.destFolder = "${WORKSPACE}/export/${appInfo[1]}" + env.srcFile = "${WORKSPACE}/${appInfo[2]}" + } + dir (env.destFolder) { + if (appInfo[2]) { + echo "Copy ${srcFile} to ${destFolder}" + if (isUnix()) { + sh 'cp \"${srcFile}\" ${destFolder}' + } + else { + bat 'copy \"%srcFile%\" %destFolder%' + } + } + } + } + } +} + +return this; \ No newline at end of file From aeb712f07339a59e87b47703c71e9e4444313871 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 13:24:11 +0000 Subject: [PATCH 031/558] Make variants of build all in one run --- build/eclipsebuild/eclipsebuild.gsh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index e7dcd09..579a413 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -1,23 +1,28 @@ // Build and export files def doBuild(apps) { for (appInfo in apps ) { - if (appInfo[0]) + env.build = appInfo[0] + env.project = appInfo[1] + env.variants = appInfo[2] + env.exportpath = appInfo[3] + if (build) { - retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) + for (build_variant in variants) { + retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh "${BUILD_COMMAND} -build ${appInfo[1]}/${BUILD_VARIANT}" + sh "${BUILD_COMMAND} -build ${project}/${build_variant}" } else { - bat "%BUILD_COMMAND% -build ${appInfo[1]}/${BUILD_VARIANT}" + bat "%BUILD_COMMAND% -build ${project}/${build_variant}" } } - echo "Looking at: ${appInfo[2]}" + echo "Export path at: ${exportpath}" script { - env.destFolder = "${WORKSPACE}/export/${appInfo[1]}" - env.srcFile = "${WORKSPACE}/${appInfo[2]}" + env.destFolder = "${WORKSPACE}/export/${project}" + env.srcFile = "${WORKSPACE}/${exportpath}" } dir (env.destFolder) { - if (appInfo[2]) { + if (exportpath) { echo "Copy ${srcFile} to ${destFolder}" if (isUnix()) { sh 'cp \"${srcFile}\" ${destFolder}' From 12a002f5a5dbe22a22255019d5ea27364005c179 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 15:35:20 +0000 Subject: [PATCH 032/558] Make variants of build all in one run --- build/eclipsebuild/eclipsebuild.gsh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 579a413..c7876d2 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -20,6 +20,7 @@ def doBuild(apps) { script { env.destFolder = "${WORKSPACE}/export/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = "${srcFile}" } dir (env.destFolder) { if (exportpath) { From ff5b8f515e7514a31e5aecbb7022c8344b81f9a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 15:43:11 +0000 Subject: [PATCH 033/558] Make variants of build all in one run --- build/eclipsebuild/eclipsebuild.gsh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index c7876d2..a4ee79a 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -20,16 +20,16 @@ def doBuild(apps) { script { env.destFolder = "${WORKSPACE}/export/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = "${srcFile}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } dir (env.destFolder) { if (exportpath) { - echo "Copy ${srcFile} to ${destFolder}" + echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh 'cp \"${srcFile}\" ${destFolder}' + sh 'cp \"${srcFileExpanded}\" ${destFolder}' } else { - bat 'copy \"%srcFile%\" %destFolder%' + bat 'copy \"%srcFileExpanded%\" %destFolder%' } } } From c21c6348a423654bb9a9d863863877b2bf4ba371 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 16:16:09 +0000 Subject: [PATCH 034/558] Make variants of build all in one run --- build/eclipsebuild/eclipsebuild.gsh | 50 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index a4ee79a..6e24393 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -1,38 +1,38 @@ // Build and export files def doBuild(apps) { for (appInfo in apps ) { - env.build = appInfo[0] - env.project = appInfo[1] - env.variants = appInfo[2] - env.exportpath = appInfo[3] - if (build) - { + build = appInfo[0] + project = appInfo[1] + variants = appInfo[2] + exportpath = appInfo[3] + if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) - if (isUnix()) { - sh "${BUILD_COMMAND} -build ${project}/${build_variant}" - } - else { - bat "%BUILD_COMMAND% -build ${project}/${build_variant}" - } - } - echo "Export path at: ${exportpath}" - script { - env.destFolder = "${WORKSPACE}/export/${project}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") - } - dir (env.destFolder) { - if (exportpath) { - echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh 'cp \"${srcFileExpanded}\" ${destFolder}' + sh "${BUILD_COMMAND} -build ${project}/${build_variant}" } else { - bat 'copy \"%srcFileExpanded%\" %destFolder%' + bat "%BUILD_COMMAND% -build ${project}/${build_variant}" } } - } + echo "Export path at: ${exportpath}" + script { + env.destFolder = "${WORKSPACE}/export/${project}" + env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + } + dir (env.destFolder) { + if (exportpath) { + echo "Copy ${srcFileExpanded} to ${destFolder}" + if (isUnix()) { + sh 'cp \"${srcFileExpanded}\" ${destFolder}' + } + else { + bat 'copy \"%srcFileExpanded%\" %destFolder%' + } + } + } + } } } } From e76809830b161294c1c832747423df3728ac033e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 16:26:19 +0000 Subject: [PATCH 035/558] Make variants of build all in one run --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 6e24393..351f9d5 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -17,7 +17,7 @@ def doBuild(apps) { } echo "Export path at: ${exportpath}" script { - env.destFolder = "${WORKSPACE}/export/${project}" + env.destFolder = "${WORKSPACE}/export/${build_variant}/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } From 75d8716a0a6ceb1e53a64984fd9765b5323cb8f8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 16:30:18 +0000 Subject: [PATCH 036/558] Make variants of build all in one run --- build/eclipsebuild/eclipsebuild.gsh | 30 +++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 351f9d5..26ed411 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -15,20 +15,22 @@ def doBuild(apps) { bat "%BUILD_COMMAND% -build ${project}/${build_variant}" } } - echo "Export path at: ${exportpath}" - script { - env.destFolder = "${WORKSPACE}/export/${build_variant}/${project}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") - } - dir (env.destFolder) { - if (exportpath) { - echo "Copy ${srcFileExpanded} to ${destFolder}" - if (isUnix()) { - sh 'cp \"${srcFileExpanded}\" ${destFolder}' - } - else { - bat 'copy \"%srcFileExpanded%\" %destFolder%' + if (exportpath) { + echo "Export path at: ${exportpath}" + script { + env.destFolder = "${WORKSPACE}/export/${build_variant}/${project}" + env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + } + dir (env.destFolder) { + if (exportpath) { + echo "Copy ${srcFileExpanded} to ${destFolder}" + if (isUnix()) { + sh 'cp \"${srcFileExpanded}\" ${destFolder}' + } + else { + bat 'copy \"%srcFileExpanded%\" %destFolder%' + } } } } From cfbc3e70c58f182d0eefd7e70f58cbc2ec5a9186 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 19 Dec 2017 18:17:15 +0000 Subject: [PATCH 037/558] Add version stamping --- versionstamping/.project | 17 ++++++++++++++ versionstamping/.pydevproject | 8 +++++++ versionstamping/versionstamper.py | 37 +++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..45c94e2 --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,37 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + filecopy = filetoconvert + ".orig" + try: + shutil.move(filetoconvert, filecopy) + except FileNotFoundError: + print("Could not find file: {0}\n".format(filetoconvert)) + return 1 + + with open(filetoconvert, 'x') as f: + with open(filecopy) as c: + converted = c.read().replace(pattern, substitute) + f.write(converted) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 64926607ca790789319e01f03ab5595f546b6eb7 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:09:34 +0000 Subject: [PATCH 038/558] REVIEW ME: Changing the versionstamp target from 1.30000.0.0 to 1.3000.0.0 so that it stamps over what the ESW is currently outputting by default. This may need to be reverted and the version of the ESW output changed. --- versionstamping/versionstamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 45c94e2..8e445d2 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -8,7 +8,7 @@ def main(argv): # Call once arguments to allow or disallow something parser.add_argument('-f', '--filetoconvert', help='The file to convert') - parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-p', '--pattern', default='1.3000.0.0', help='The pattern to search for and replace') parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') args = parser.parse_args(argv) From 3c1b453b21ee7bb07b8e93f31ca7a4bf118b206c Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:16:24 +0000 Subject: [PATCH 039/558] Reverting change as i have realized the blunder --- versionstamping/versionstamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 8e445d2..45c94e2 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -8,7 +8,7 @@ def main(argv): # Call once arguments to allow or disallow something parser.add_argument('-f', '--filetoconvert', help='The file to convert') - parser.add_argument('-p', '--pattern', default='1.3000.0.0', help='The pattern to search for and replace') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') args = parser.parse_args(argv) From f3107d5dcfc9cee65a8ac3be876611cea2b417dd Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:35:58 +0000 Subject: [PATCH 040/558] Adding Debugging for conversion script, Reverting the conversion values for build.gsh --- versionstamping/versionstamper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 45c94e2..ff5eb0d 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -28,7 +28,9 @@ def main(argv): with open(filetoconvert, 'x') as f: with open(filecopy) as c: + print("Detected %s instances of %s pre-conversion"%(c.count(pattern),pattern)) converted = c.read().replace(pattern, substitute) + print("Confirmed %s instances of %s post-conversion"%s(converted.count(substitute),substitute)) f.write(converted) return 0 From 35cbc075a36d1a0ed8f83e8a7ef4651f23c5ef79 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:37:44 +0000 Subject: [PATCH 041/558] Read the file before attempting to count instances within it. --- versionstamping/versionstamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index ff5eb0d..8b43e48 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -28,7 +28,7 @@ def main(argv): with open(filetoconvert, 'x') as f: with open(filecopy) as c: - print("Detected %s instances of %s pre-conversion"%(c.count(pattern),pattern)) + print("Detected %s instances of %s pre-conversion"%(c.read().count(pattern),pattern)) converted = c.read().replace(pattern, substitute) print("Confirmed %s instances of %s post-conversion"%s(converted.count(substitute),substitute)) f.write(converted) From 58feb9612024208857ecc670294a931876d55949 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:38:48 +0000 Subject: [PATCH 042/558] silly 's' --- versionstamping/versionstamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 8b43e48..48d9f45 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -30,7 +30,7 @@ def main(argv): with open(filecopy) as c: print("Detected %s instances of %s pre-conversion"%(c.read().count(pattern),pattern)) converted = c.read().replace(pattern, substitute) - print("Confirmed %s instances of %s post-conversion"%s(converted.count(substitute),substitute)) + print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) f.write(converted) return 0 From 1124ca5cd683fa690db227c97d7b9e458c1f0e55 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:51:45 +0000 Subject: [PATCH 043/558] close the files after we have read/written them. explicily open copy file for reading --- versionstamping/versionstamper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 48d9f45..f42e54d 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -27,11 +27,13 @@ def main(argv): return 1 with open(filetoconvert, 'x') as f: - with open(filecopy) as c: + with open(filecopy,'r') as c: print("Detected %s instances of %s pre-conversion"%(c.read().count(pattern),pattern)) converted = c.read().replace(pattern, substitute) print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) f.write(converted) + c.close() + f.close() return 0 From 9b75cd07247ae0d7908a709611a795709593928e Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:52:52 +0000 Subject: [PATCH 044/558] Print Converted as debug info. --- versionstamping/versionstamper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index f42e54d..d953aa4 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -30,6 +30,7 @@ def main(argv): with open(filecopy,'r') as c: print("Detected %s instances of %s pre-conversion"%(c.read().count(pattern),pattern)) converted = c.read().replace(pattern, substitute) + print ("DEBUG: %s "%converted) print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) f.write(converted) c.close() From 42bdcc82e2e35fa367d2e7cf66ed6490cc07fc43 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 15:58:54 +0000 Subject: [PATCH 045/558] Read to local variable rather than double reading from file. --- versionstamping/versionstamper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index d953aa4..da01aa2 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -28,12 +28,13 @@ def main(argv): with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: - print("Detected %s instances of %s pre-conversion"%(c.read().count(pattern),pattern)) - converted = c.read().replace(pattern, substitute) + fileread = c.read() + c.close() + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) print ("DEBUG: %s "%converted) print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) f.write(converted) - c.close() f.close() return 0 From ae37de4bc32d1ca82b254d7a9cda94319b541e41 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 16:00:08 +0000 Subject: [PATCH 046/558] Comment Debug Print --- versionstamping/versionstamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index da01aa2..ebfad73 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -32,7 +32,7 @@ def main(argv): c.close() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) - print ("DEBUG: %s "%converted) + #print ("DEBUG: %s "%converted) print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) f.write(converted) f.close() From 3c0896f1ee63e63d11c065e8836c9725fbf5b8d2 Mon Sep 17 00:00:00 2001 From: georgelinsdell Date: Wed, 20 Dec 2017 16:32:16 +0000 Subject: [PATCH 047/558] Remove File.close() calls as they are not neccessary --- versionstamping/versionstamper.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index ebfad73..3e59c6f 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -29,13 +29,11 @@ def main(argv): with open(filetoconvert, 'x') as f: with open(filecopy,'r') as c: fileread = c.read() - c.close() print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) #print ("DEBUG: %s "%converted) print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) f.write(converted) - f.close() return 0 From 82b7ef85bd50d57db1b3bdfe458430ca2c5ab8a8 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Fri, 12 Jan 2018 15:57:39 +0000 Subject: [PATCH 048/558] Update versioninfo.py so that default prefix ends with a period (1.30000.0.) --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index a4162e0..59b5225 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -68,7 +68,7 @@ def getGitRevision(gitrootpath, basenumber): def getVersionPrefix(sourcebranch): print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - defaultversionprefix = "1.30000.0" + defaultversionprefix = "1.30000.0." splits = sourcebranch.split('/') if len(splits) < 1: print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) From 5b6835d1174b29d909eab8f2b8d0d92d649a3db8 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 09:56:48 +0000 Subject: [PATCH 049/558] Accounting for project names containing spaces. --- build/eclipsebuild/eclipsebuild.gsh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 26ed411..5d55fe5 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -1,18 +1,19 @@ // Build and export files def doBuild(apps) { for (appInfo in apps ) { - build = appInfo[0] - project = appInfo[1] - variants = appInfo[2] - exportpath = appInfo[3] + (build, project, variants, exportpath) = appInfo if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh "${BUILD_COMMAND} -build ${project}/${build_variant}" + sh """ + "${BUILD_COMMAND}" -build "${project}/${build_variant}" + """ } else { - bat "%BUILD_COMMAND% -build ${project}/${build_variant}" + bat """ + "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + """ } } if (exportpath) { @@ -39,4 +40,4 @@ def doBuild(apps) { } } -return this; \ No newline at end of file +return this; From 22d283a9985c4ea16ce1ea0af4235f04bb7b25dd Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 09:56:48 +0000 Subject: [PATCH 050/558] Accounting for project names containing spaces. --- build/eclipsebuild/eclipsebuild.gsh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 26ed411..5d55fe5 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -1,18 +1,19 @@ // Build and export files def doBuild(apps) { for (appInfo in apps ) { - build = appInfo[0] - project = appInfo[1] - variants = appInfo[2] - exportpath = appInfo[3] + (build, project, variants, exportpath) = appInfo if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh "${BUILD_COMMAND} -build ${project}/${build_variant}" + sh """ + "${BUILD_COMMAND}" -build "${project}/${build_variant}" + """ } else { - bat "%BUILD_COMMAND% -build ${project}/${build_variant}" + bat """ + "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + """ } } if (exportpath) { @@ -39,4 +40,4 @@ def doBuild(apps) { } } -return this; \ No newline at end of file +return this; From 2a3ad3e0051d0357cd51b38cf5e647481d8f9a9b Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 10:39:40 +0000 Subject: [PATCH 051/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 5d55fe5..da349c6 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -7,7 +7,7 @@ def doBuild(apps) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { sh """ - "${BUILD_COMMAND}" -build "${project}/${build_variant}" + \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\" """ } else { From 74e7696ef6dc4f748e810de6300b8ad1d737164f Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 10:39:40 +0000 Subject: [PATCH 052/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 5d55fe5..da349c6 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -7,7 +7,7 @@ def doBuild(apps) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { sh """ - "${BUILD_COMMAND}" -build "${project}/${build_variant}" + \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\" """ } else { From ede80e31d3833bead3b20fa86437e129df4d5e01 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 10:53:00 +0000 Subject: [PATCH 053/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index da349c6..b4bbc34 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,10 +6,8 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh """ - \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\" - """ - } + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + } else { bat """ "%BUILD_COMMAND%" -build "${project}\\${build_variant}" From 7d8891e227f95361fdc0a2b80b3c1cb114053525 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 10:53:00 +0000 Subject: [PATCH 054/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index da349c6..b4bbc34 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,10 +6,8 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh """ - \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\" - """ - } + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + } else { bat """ "%BUILD_COMMAND%" -build "${project}\\${build_variant}" From 65c3bdf95cd8bfd30e28cf42aa365fc08418e871 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 10:56:32 +0000 Subject: [PATCH 055/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index b4bbc34..0ba188d 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,7 +6,7 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + sh '${BUILD_COMMAND} -build "${project}/${build_variant}"' } else { bat """ From ec0055e572f26eedfbd33134a2da4a1d403e04eb Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 10:56:32 +0000 Subject: [PATCH 056/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index b4bbc34..0ba188d 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,7 +6,7 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + sh '${BUILD_COMMAND} -build "${project}/${build_variant}"' } else { bat """ From 60a33d6a8a7407773767de17eac872bf205edb53 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:02:51 +0000 Subject: [PATCH 057/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 0ba188d..b4bbc34 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,7 +6,7 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh '${BUILD_COMMAND} -build "${project}/${build_variant}"' + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" } else { bat """ From 61d581de84a5cef38c55e2e345f20f9b447b9130 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:02:51 +0000 Subject: [PATCH 058/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 0ba188d..b4bbc34 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,7 +6,7 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - sh '${BUILD_COMMAND} -build "${project}/${build_variant}"' + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" } else { bat """ From 69904f1efd05096fda2f22d83fad2f1f912ff4a0 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:08:10 +0000 Subject: [PATCH 059/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index b4bbc34..367b454 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,6 +6,7 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { + echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" } else { From 4400c698df7a556eca83fe1d474c0d066486f6d6 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:08:10 +0000 Subject: [PATCH 060/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index b4bbc34..367b454 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,6 +6,7 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { + echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" } else { From 3f61ee3e423a0a7680260ca578e73d345d0cd818 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:29:10 +0000 Subject: [PATCH 061/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 367b454..5384341 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,13 +6,11 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + echo "Build command: \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" + sh "\"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" } else { - bat """ - "%BUILD_COMMAND%" -build "${project}\\${build_variant}" - """ + bat "\"%BUILD_COMMAND%\" -build \"${project}/${build_variant}\"" } } if (exportpath) { From c55ffd413675729698bcf1467d25ba05306bc3b6 Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:29:10 +0000 Subject: [PATCH 062/558] Auto build fix. --- build/eclipsebuild/eclipsebuild.gsh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 367b454..5384341 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,13 +6,11 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + echo "Build command: \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" + sh "\"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" } else { - bat """ - "%BUILD_COMMAND%" -build "${project}\\${build_variant}" - """ + bat "\"%BUILD_COMMAND%\" -build \"${project}/${build_variant}\"" } } if (exportpath) { From c3008ea17db8df74dde9aea8d33bfc648e902e9f Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:40:39 +0000 Subject: [PATCH 063/558] Revert "Auto build fix." This reverts commit 29e58eead620367c9beccb69b35c6715a5c2ef33. --- build/eclipsebuild/eclipsebuild.gsh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 5384341..367b454 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,11 +6,13 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - echo "Build command: \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" - sh "\"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" + echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" } else { - bat "\"%BUILD_COMMAND%\" -build \"${project}/${build_variant}\"" + bat """ + "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + """ } } if (exportpath) { From a21ec9aace7535a45026110a1fd008fdadd4060a Mon Sep 17 00:00:00 2001 From: danielbujnik Date: Tue, 13 Feb 2018 11:40:39 +0000 Subject: [PATCH 064/558] Revert "Auto build fix." This reverts commit 29e58eead620367c9beccb69b35c6715a5c2ef33. --- build/eclipsebuild/eclipsebuild.gsh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 5384341..367b454 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -6,11 +6,13 @@ def doBuild(apps) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - echo "Build command: \"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" - sh "\"${BUILD_COMMAND}\" -build \"${project}/${build_variant}\"" + echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" } else { - bat "\"%BUILD_COMMAND%\" -build \"${project}/${build_variant}\"" + bat """ + "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + """ } } if (exportpath) { From 70b23d7049b5a986935164e72e506d17c7de1903 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 17:03:14 +0000 Subject: [PATCH 065/558] Move commonly used methods out to Common_scripts --- commandline/commandline.gsh | 12 ++++++++ configuration/configuration.gsh | 34 +++++++++++++++++++++ versioncontrol/git.gsh | 53 +++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 commandline/commandline.gsh create mode 100644 configuration/configuration.gsh create mode 100644 versioncontrol/git.gsh diff --git a/commandline/commandline.gsh b/commandline/commandline.gsh new file mode 100644 index 0000000..c03add7 --- /dev/null +++ b/commandline/commandline.gsh @@ -0,0 +1,12 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return bat(returnStdout: true, script: """ + @echo off + ${cmd} + """ + ).trim() + } +} + diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh new file mode 100644 index 0000000..424765e --- /dev/null +++ b/configuration/configuration.gsh @@ -0,0 +1,34 @@ +def githelper = load("Common_scripts/versioncontrol/git.gsh") + +configurationCollection = [:] + +def registerConfigurationInfo(String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + def ConfigurationInfo = [configurationURL, configurationVersion, configurationSHA] + configurationCollection.put(configurationName, ConfigurationInfo) +} + +def registerPipelineScriptConfiguration() { + registerConfigurationInfo('pipeline script', '-', '-', getGitInfo("${WORKSPACE}")) +} + +def saveConfigurationInVersionInfo() { + dir("${WORKSPACE}\\version") { + writeFile file: 'configuration.txt', text: getFormattedConfiguration() + } +} + +String getFormattedConfiguration() { + String formattedInfo = "" + def keys = configurationCollection.keySet().sort() + for (key in keys) { + (configurationURL, configurationVersion, configurationSHA) = configurationCollection.get(key) + formattedLine = String.format("%s:%s,%s,%s%n", key, configurationURL, configurationVersion, configurationSHA) + formattedInfo += formattedLine + } + return formattedInfo +} + +def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + githelper.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + registerConfigurationInfo(tool, gitURL, revision, getGitInfo(targetDir)) +} diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh new file mode 100644 index 0000000..1f6d88e --- /dev/null +++ b/versioncontrol/git.gsh @@ -0,0 +1,53 @@ +def commandlinehelper = load("Common_scripts/commandline/commandline.gsh") + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return commandlinehelper.getCommandOutput('git log -1 --format=oneline') + } +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: "${REPO_ROOT}/${repoRoot}", shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 60], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + ], + submoduleCfg: [], + ]) +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() <= 1) { + return false + } else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} From ca2db55d2bb2a292e1bc070eceff75f6c4721174 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 17:16:02 +0000 Subject: [PATCH 066/558] Move commonly used methods out to Common_scripts --- configuration/configuration.gsh | 4 ++-- versioncontrol/git.gsh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 424765e..1f86c42 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -def githelper = load("Common_scripts/versioncontrol/git.gsh") +def gh = load("${WORKSPACE}/Common_scripts/versioncontrol/git.gsh") configurationCollection = [:] @@ -29,6 +29,6 @@ String getFormattedConfiguration() { } def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { - githelper.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) + gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) registerConfigurationInfo(tool, gitURL, revision, getGitInfo(targetDir)) } diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 1f6d88e..90dc7ee 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,8 +1,8 @@ -def commandlinehelper = load("Common_scripts/commandline/commandline.gsh") +def clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { - return commandlinehelper.getCommandOutput('git log -1 --format=oneline') + return clh.getCommandOutput('git log -1 --format=oneline') } } From 1e0f22ed5e86d8e0765b764c7cedea1522280c98 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 17:19:29 +0000 Subject: [PATCH 067/558] Move commonly used methods out to Common_scripts --- commandline/commandline.gsh | 1 + 1 file changed, 1 insertion(+) diff --git a/commandline/commandline.gsh b/commandline/commandline.gsh index c03add7..df99a75 100644 --- a/commandline/commandline.gsh +++ b/commandline/commandline.gsh @@ -10,3 +10,4 @@ String getCommandOutput(String cmd) { } } +return this \ No newline at end of file From 4c6372bb8c06143a4a017e02368a40cfd53eb03c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 17:22:49 +0000 Subject: [PATCH 068/558] Move commonly used methods out to Common_scripts --- configuration/configuration.gsh | 2 ++ versioncontrol/git.gsh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 1f86c42..aa732e6 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -32,3 +32,5 @@ def gitCheckoutAndRegister(String tool, String gitURL, String revision, String t gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) registerConfigurationInfo(tool, gitURL, revision, getGitInfo(targetDir)) } + +return this diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 90dc7ee..06e5b50 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -51,3 +51,5 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { } return true } + +return this \ No newline at end of file From a1448ac8509c0b512092295700f1f0a6f61bba27 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 17:27:26 +0000 Subject: [PATCH 069/558] Move commonly used methods out to Common_scripts --- configuration/configuration.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index aa732e6..976ebf8 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -8,7 +8,7 @@ def registerConfigurationInfo(String configurationName, String configurationURL, } def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', getGitInfo("${WORKSPACE}")) + registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${WORKSPACE}")) } def saveConfigurationInVersionInfo() { @@ -30,7 +30,7 @@ String getFormattedConfiguration() { def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(tool, gitURL, revision, getGitInfo(targetDir)) + registerConfigurationInfo(tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this From d09d20c7b663d7a62cad3130ee91a05a96ab5f4a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 17:30:21 +0000 Subject: [PATCH 070/558] Move commonly used methods out to Common_scripts --- configuration/configuration.gsh | 2 +- versioncontrol/git.gsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 976ebf8..29cd7e3 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -def gh = load("${WORKSPACE}/Common_scripts/versioncontrol/git.gsh") +gh = load("${WORKSPACE}/Common_scripts/versioncontrol/git.gsh") configurationCollection = [:] diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 06e5b50..fd633eb 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,4 +1,4 @@ -def clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") +clh = load("${WORKSPACE}/Common_scripts/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 9d32414fca002f779a7d924953dcf4cd7b7778a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 18:16:20 +0000 Subject: [PATCH 071/558] Check for undefined REPO_ROOT environment variable --- versioncontrol/git.gsh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index fd633eb..145045c 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -7,6 +7,13 @@ String getGitInfo(String workingSetRootPath) { } def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + reporootenv = env.REPO_ROOT + if (reporootenv != mull && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -18,7 +25,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo doGenerateSubmoduleConfigurations: false, extensions: [ [$class: 'CleanBeforeCheckout'], - [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: "${REPO_ROOT}/${repoRoot}", shallow: shallowClone, timeout: 120], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 60], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] ], From 69be84aac0e12da69f55d2c5b01f7ad22f775922 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Feb 2018 18:17:26 +0000 Subject: [PATCH 072/558] Check for undefined REPO_ROOT environment variable --- versioncontrol/git.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 145045c..70fe6bc 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -8,7 +8,7 @@ String getGitInfo(String workingSetRootPath) { def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { reporootenv = env.REPO_ROOT - if (reporootenv != mull && repoRoot != "") { + if (reporootenv != null && repoRoot != "") { reporootpath = "${reporootenv.trim()}/${repoRoot}" } else { reporootpath = "" From d1a8c098a6ba03fe5461f82db036de2f48aa2c62 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Feb 2018 17:50:05 +0000 Subject: [PATCH 073/558] Make the version number 1.20000.x if the branch name is .x or ..x --- versionnumbering/versioninfo.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 59b5225..45fd1ca 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -65,10 +65,14 @@ def getGitRevision(gitrootpath, basenumber): return (adjustedchangecount, revision, remoteinfo) -def getVersionPrefix(sourcebranch): +# Handle: +# origin/master +# origin/2.0.x + def getVersionPrefix(sourcebranch): print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." splits = sourcebranch.split('/') if len(splits) < 1: print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) @@ -77,7 +81,7 @@ def getVersionPrefix(sourcebranch): sourcebranch = splits[-1] if sourcebranch.lower().endswith("master"): # Special case - return "1.20000.0." + return mastertypeversionprefix splits = sourcebranch.split('.') versionprefix = "" @@ -86,21 +90,27 @@ def getVersionPrefix(sourcebranch): versionprefix = defaultversionprefix else: splitnumbercount = 0 + isIntermediateReleaseBranch = false for split in splits: if split.isdigit(): versionprefix += split + "." splitnumbercount += 1 else: + if split == "x": + isIntermediateReleaseBranch = true break if splitnumbercount > 2: break if splitnumbercount == 0: print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) versionprefix = defaultversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix From 061b3d4fcfb9fe3f0a853b63cf6ea48ea8f1ce0b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Feb 2018 18:14:49 +0000 Subject: [PATCH 074/558] Make the version number 1.20000.x if the branch name is .x or ..x --- .pydevproject | 2 +- versionnumbering/versioninfo.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pydevproject b/.pydevproject index d111c07..1ce5929 100644 --- a/.pydevproject +++ b/.pydevproject @@ -3,6 +3,6 @@ /${PROJECT_DIR_NAME} -python interpreter +python 2.7 python35 diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 45fd1ca..d9e6681 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -68,7 +68,7 @@ def getGitRevision(gitrootpath, basenumber): # Handle: # origin/master # origin/2.0.x - def getVersionPrefix(sourcebranch): +def getVersionPrefix(sourcebranch): print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) defaultversionprefix = "1.30000.0." @@ -90,14 +90,14 @@ def getGitRevision(gitrootpath, basenumber): versionprefix = defaultversionprefix else: splitnumbercount = 0 - isIntermediateReleaseBranch = false + isIntermediateReleaseBranch = False for split in splits: if split.isdigit(): versionprefix += split + "." splitnumbercount += 1 else: if split == "x": - isIntermediateReleaseBranch = true + isIntermediateReleaseBranch = True break if splitnumbercount > 2: break From 9af8da5da65ee4420ce6394e5c34c3fedf9fef8f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Feb 2018 11:13:49 +0000 Subject: [PATCH 075/558] gitrevision.txt and revision.txt file content were mixed up with previous way of generating version folder --- versionnumbering/versioninfo.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index d9e6681..25644a6 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -115,7 +115,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, remoteinfo, buildnumber, buildstamp): +def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp): with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: f.write(buildjobname) @@ -123,6 +123,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, r f.write(versionprefix) with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(gitrevisioncount) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: f.write(gitrevision) with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: @@ -195,7 +198,7 @@ USAGE buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevision, remoteinfo, buildnumber, buildstamp) + writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, buildnumber, buildstamp) return 0 except Exception as e: From d3ba4321ae5b3e3bedf77ec23d232a66b927c7f8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Feb 2018 11:38:11 +0000 Subject: [PATCH 076/558] The print server base revision number is 5000 --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 25644a6..9a7cd6a 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -123,7 +123,7 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou f.write(versionprefix) with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(gitrevisioncount) + f.write(str(gitrevisioncount)) with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: f.write(gitrevision) From 96bba5d019bd9ee913af8abc5a930ba086f3d513 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 21 Feb 2018 13:28:27 +0000 Subject: [PATCH 077/558] Provide a parameter for deciding where files are exported --- build/eclipsebuild/eclipsebuild.gsh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 367b454..30fe4d5 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -1,24 +1,24 @@ // Build and export files -def doBuild(apps) { +def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { (build, project, variants, exportpath) = appInfo if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - } + } else { bat """ - "%BUILD_COMMAND%" -build "${project}\\${build_variant}" - """ + "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + """ } } if (exportpath) { echo "Export path at: ${exportpath}" script { - env.destFolder = "${WORKSPACE}/export/${build_variant}/${project}" + env.destFolder = "${exportfolder}/${build_variant}/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } From 99bb2d6465220a29e7a3dfd7031dd49454f5013f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 21 Feb 2018 13:28:27 +0000 Subject: [PATCH 078/558] Provide a parameter for deciding where files are exported --- build/eclipsebuild/eclipsebuild.gsh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/build/eclipsebuild/eclipsebuild.gsh index 367b454..30fe4d5 100644 --- a/build/eclipsebuild/eclipsebuild.gsh +++ b/build/eclipsebuild/eclipsebuild.gsh @@ -1,24 +1,24 @@ // Build and export files -def doBuild(apps) { +def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { (build, project, variants, exportpath) = appInfo if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { - echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - } + } else { bat """ - "%BUILD_COMMAND%" -build "${project}\\${build_variant}" - """ + "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + """ } } if (exportpath) { echo "Export path at: ${exportpath}" script { - env.destFolder = "${WORKSPACE}/export/${build_variant}/${project}" + env.destFolder = "${exportfolder}/${build_variant}/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } From 1f266505b15e668a63d58fd67b1a6b1d01e42f05 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 21 Feb 2018 16:47:21 +0000 Subject: [PATCH 079/558] Configuration information was not being written into version folder on linux machine --- configuration/configuration.gsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 29cd7e3..b91e0a3 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -12,7 +12,8 @@ def registerPipelineScriptConfiguration() { } def saveConfigurationInVersionInfo() { - dir("${WORKSPACE}\\version") { + dir("${WORKSPACE}/version") { + echo "Writing configuration info to ${WORKSPACE}/version/configuration.txt" writeFile file: 'configuration.txt', text: getFormattedConfiguration() } } From 96ac5a636249e5ccb392cc2c78755de22afd4044 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 21 Feb 2018 16:47:21 +0000 Subject: [PATCH 080/558] Configuration information was not being written into version folder on linux machine --- configuration/configuration.gsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 29cd7e3..b91e0a3 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -12,7 +12,8 @@ def registerPipelineScriptConfiguration() { } def saveConfigurationInVersionInfo() { - dir("${WORKSPACE}\\version") { + dir("${WORKSPACE}/version") { + echo "Writing configuration info to ${WORKSPACE}/version/configuration.txt" writeFile file: 'configuration.txt', text: getFormattedConfiguration() } } From 049b242a2a93c3e8efb4037ca6b82c3759b1baf6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Feb 2018 13:00:31 +0000 Subject: [PATCH 081/558] Change folders and files to match convention --- .../eclipsebuild.gsh => compile/eclipsebuild/eclipseds5build.gsh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/eclipsebuild/eclipsebuild.gsh => compile/eclipsebuild/eclipseds5build.gsh (100%) diff --git a/build/eclipsebuild/eclipsebuild.gsh b/compile/eclipsebuild/eclipseds5build.gsh similarity index 100% rename from build/eclipsebuild/eclipsebuild.gsh rename to compile/eclipsebuild/eclipseds5build.gsh From 17eae1e1357f6e7d3361639dc3bf184bd308f9bc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Feb 2018 09:17:05 +0000 Subject: [PATCH 082/558] Save the slave machine name in the version folder --- versionnumbering/versioninfo.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 9a7cd6a..a9d8738 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -22,6 +22,7 @@ __date__ = '2017-11-29' __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") +slaveMachine = os.getenv("NODE_NAME", default="-") def resetFolder(versionfolder): print("Removing folders and contents of: {0}".format(versionfolder)) @@ -141,6 +142,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: f.write(buildstampcommasep) + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + def main(argv=None): # IGNORE:C0111 '''Command line options.''' From 0fd3b357a5c46a2146adb90dab356b4767a959f0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 15:22:13 +0100 Subject: [PATCH 083/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 52 +++++++++++++++--------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 30fe4d5..e17f4b9 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,7 +1,34 @@ + +def doExport(exportpathlist, build_variant = "") { + for (exportpath in exportpathlist) { + echo "Export path at: ${exportpath}" + script { + if (build_variant) { + env.destFolder = "${exportfolder}/${build_variant}/${project}" + env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + } + else { + env.destFolder = "${exportfolder}/${project}" + env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + } + } + dir (env.destFolder) { + echo "Copy ${srcFileExpanded} to ${destFolder}" + if (isUnix()) { + sh 'cp \"${srcFileExpanded}\" ${destFolder}' + } + else { + bat 'copy \"%srcFileExpanded%\" %destFolder%' + } + } + } +} + // Build and export files def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, project, variants, exportpath) = appInfo + (build, project, variants, exportpathlist) = appInfo if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) @@ -15,27 +42,12 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { """ } } - if (exportpath) { - echo "Export path at: ${exportpath}" - script { - env.destFolder = "${exportfolder}/${build_variant}/${project}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") - } - dir (env.destFolder) { - if (exportpath) { - echo "Copy ${srcFileExpanded} to ${destFolder}" - if (isUnix()) { - sh 'cp \"${srcFileExpanded}\" ${destFolder}' - } - else { - bat 'copy \"%srcFileExpanded%\" %destFolder%' - } - } - } - } + doExport(exportpathlist, build_variant) } } + else { + doExport(exportpathlist, build_variant) + } } } From 423e88d3221238b488c2c1b9eeaa6bfe77d4f383 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 15:22:13 +0100 Subject: [PATCH 084/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 52 +++++++++++++++--------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 30fe4d5..e17f4b9 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,7 +1,34 @@ + +def doExport(exportpathlist, build_variant = "") { + for (exportpath in exportpathlist) { + echo "Export path at: ${exportpath}" + script { + if (build_variant) { + env.destFolder = "${exportfolder}/${build_variant}/${project}" + env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + } + else { + env.destFolder = "${exportfolder}/${project}" + env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + } + } + dir (env.destFolder) { + echo "Copy ${srcFileExpanded} to ${destFolder}" + if (isUnix()) { + sh 'cp \"${srcFileExpanded}\" ${destFolder}' + } + else { + bat 'copy \"%srcFileExpanded%\" %destFolder%' + } + } + } +} + // Build and export files def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, project, variants, exportpath) = appInfo + (build, project, variants, exportpathlist) = appInfo if (build) { for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) @@ -15,27 +42,12 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { """ } } - if (exportpath) { - echo "Export path at: ${exportpath}" - script { - env.destFolder = "${exportfolder}/${build_variant}/${project}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") - } - dir (env.destFolder) { - if (exportpath) { - echo "Copy ${srcFileExpanded} to ${destFolder}" - if (isUnix()) { - sh 'cp \"${srcFileExpanded}\" ${destFolder}' - } - else { - bat 'copy \"%srcFileExpanded%\" %destFolder%' - } - } - } - } + doExport(exportpathlist, build_variant) } } + else { + doExport(exportpathlist, build_variant) + } } } From 3065c8d9b9e9a9972f3152c9c7ce745d2fcb74de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 15:34:48 +0100 Subject: [PATCH 085/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index e17f4b9..065e64a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -34,8 +34,8 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - } + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + } else { bat """ "%BUILD_COMMAND%" -build "${project}\\${build_variant}" @@ -46,7 +46,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } else { - doExport(exportpathlist, build_variant) + doExport(exportpathlist) } } } From a34f8ee1e896397e4e1eb7a7ca5331c07b748878 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 15:34:48 +0100 Subject: [PATCH 086/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index e17f4b9..065e64a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -34,8 +34,8 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) if (isUnix()) { echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - } + sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + } else { bat """ "%BUILD_COMMAND%" -build "${project}\\${build_variant}" @@ -46,7 +46,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } else { - doExport(exportpathlist, build_variant) + doExport(exportpathlist) } } } From 4d3c9f6bc61833e67a5dee4d662056bb4d8c5278 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 15:47:21 +0100 Subject: [PATCH 087/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 065e64a..f0700b4 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,5 +1,5 @@ -def doExport(exportpathlist, build_variant = "") { +def doExport(exportfolder, exportpathlist, build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { @@ -42,11 +42,11 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { """ } } - doExport(exportpathlist, build_variant) + doExport(exportfolder, exportpathlist, build_variant) } } else { - doExport(exportpathlist) + doExport(exportfolder, exportpathlist) } } } From 07a0a49d06993d2423a87e5c9bbef438e1cce3ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 15:47:21 +0100 Subject: [PATCH 088/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 065e64a..f0700b4 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,5 +1,5 @@ -def doExport(exportpathlist, build_variant = "") { +def doExport(exportfolder, exportpathlist, build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { @@ -42,11 +42,11 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { """ } } - doExport(exportpathlist, build_variant) + doExport(exportfolder, exportpathlist, build_variant) } } else { - doExport(exportpathlist) + doExport(exportfolder, exportpathlist) } } } From cf844016bcc19a9d7420f908247f0590464541cc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:06:33 +0100 Subject: [PATCH 089/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index f0700b4..ca4fd2c 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,7 +16,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh 'cp \"${srcFileExpanded}\" ${destFolder}' + sh 'cp -v \"${srcFileExpanded}\" ${destFolder}' } else { bat 'copy \"%srcFileExpanded%\" %destFolder%' From 783c61598d02d01722eb6f50812693a6b4683431 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:06:33 +0100 Subject: [PATCH 090/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index f0700b4..ca4fd2c 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,7 +16,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh 'cp \"${srcFileExpanded}\" ${destFolder}' + sh 'cp -v \"${srcFileExpanded}\" ${destFolder}' } else { bat 'copy \"%srcFileExpanded%\" %destFolder%' From 6e55704be9cf8e88dfca50564712c00df65894d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:26:27 +0100 Subject: [PATCH 091/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index ca4fd2c..0801386 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,10 +16,13 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh 'cp -v \"${srcFileExpanded}\" ${destFolder}' + sh """ + echo cp -v -\"${srcFileExpanded}\"- -\"${destFolder}\"- + cp -v \"${srcFileExpanded}\" \"${destFolder}\" + """ } else { - bat 'copy \"%srcFileExpanded%\" %destFolder%' + bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' } } } From e77d9f03a1da9ae10211c8120f1947e916413e73 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:26:27 +0100 Subject: [PATCH 092/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index ca4fd2c..0801386 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,10 +16,13 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh 'cp -v \"${srcFileExpanded}\" ${destFolder}' + sh """ + echo cp -v -\"${srcFileExpanded}\"- -\"${destFolder}\"- + cp -v \"${srcFileExpanded}\" \"${destFolder}\" + """ } else { - bat 'copy \"%srcFileExpanded%\" %destFolder%' + bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' } } } From 50af99352789d1c257d5177f6338af7cb9ac87ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:43:32 +0100 Subject: [PATCH 093/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 0801386..357e64a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,10 +16,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh """ - echo cp -v -\"${srcFileExpanded}\"- -\"${destFolder}\"- - cp -v \"${srcFileExpanded}\" \"${destFolder}\" - """ + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From c937856bfd2d3a5ca3258ecf43717d522315e247 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:43:32 +0100 Subject: [PATCH 094/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 0801386..357e64a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,10 +16,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh """ - echo cp -v -\"${srcFileExpanded}\"- -\"${destFolder}\"- - cp -v \"${srcFileExpanded}\" \"${destFolder}\" - """ + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From f5cecd05c51dfafe9f3d624f652d976b57a37c22 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:55:28 +0100 Subject: [PATCH 095/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 357e64a..477d68d 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,7 +16,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\" + sh 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 142e5955443a97e25d2410689e0f9356a76219b0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 16:55:28 +0100 Subject: [PATCH 096/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 357e64a..477d68d 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -16,7 +16,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy ${srcFileExpanded} to ${destFolder}" if (isUnix()) { - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\" + sh 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From c2aa61e21b64e877572bac9cfba4a7b7011f67c1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 17:22:08 +0100 Subject: [PATCH 097/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 477d68d..fc8ecd0 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -14,9 +14,9 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { } } dir (env.destFolder) { - echo "Copy ${srcFileExpanded} to ${destFolder}" + echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' + bash 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 8c63eee9697553883202fb26e1cc9ef97a4ff469 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 17:22:08 +0100 Subject: [PATCH 098/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 477d68d..fc8ecd0 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -14,9 +14,9 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { } } dir (env.destFolder) { - echo "Copy ${srcFileExpanded} to ${destFolder}" + echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' + bash 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From c0f941f390ce39af39052fd7aed9e28c713ca1e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 17:25:12 +0100 Subject: [PATCH 099/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index fc8ecd0..b91e487 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -14,9 +14,11 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { } } dir (env.destFolder) { + echo 'Copy \"${srcFileExpanded}\" to \"${destFolder}\"' echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - bash 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' + echo "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 585ddabcf5570d1f1c7133d74fce7524596283e4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 17:25:12 +0100 Subject: [PATCH 100/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index fc8ecd0..b91e487 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -14,9 +14,11 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { } } dir (env.destFolder) { + echo 'Copy \"${srcFileExpanded}\" to \"${destFolder}\"' echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - bash 'cp -v \"${srcFileExpanded}\" \"${destFolder}\"' + echo "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 9a8a5d5be9df6bcadc61006c012a2718d10aebb8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 17:50:08 +0100 Subject: [PATCH 101/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index b91e487..9544faf 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -15,10 +15,8 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { } dir (env.destFolder) { echo 'Copy \"${srcFileExpanded}\" to \"${destFolder}\"' - echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - echo "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -v ${srcFileExpanded} \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From ec69c45d90ce01745dd5c828e0292471f0f4054b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 17:50:08 +0100 Subject: [PATCH 102/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index b91e487..9544faf 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -15,10 +15,8 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { } dir (env.destFolder) { echo 'Copy \"${srcFileExpanded}\" to \"${destFolder}\"' - echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - echo "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -v ${srcFileExpanded} \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 9f58f571b3a0c1d61b31d7898577ce65c71c14ba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 18:30:39 +0100 Subject: [PATCH 103/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9544faf..5b113d3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -3,20 +3,23 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { + echo "Build variant is ${build_variant}" if (build_variant) { + echo "Build variant is ${build_variant}" env.destFolder = "${exportfolder}/${build_variant}/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { + echo "No build variant" env.destFolder = "${exportfolder}/${project}" env.srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { - echo 'Copy \"${srcFileExpanded}\" to \"${destFolder}\"' + echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v ${srcFileExpanded} \"${destFolder}\"" + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 20a392db61328799b2357b2b38dff56ed95566b8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 18:30:39 +0100 Subject: [PATCH 104/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9544faf..5b113d3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -3,20 +3,23 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { + echo "Build variant is ${build_variant}" if (build_variant) { + echo "Build variant is ${build_variant}" env.destFolder = "${exportfolder}/${build_variant}/${project}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { + echo "No build variant" env.destFolder = "${exportfolder}/${project}" env.srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { - echo 'Copy \"${srcFileExpanded}\" to \"${destFolder}\"' + echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v ${srcFileExpanded} \"${destFolder}\"" + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From f91450c701b9362f96af554203bcace59f5190f2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 18:56:17 +0100 Subject: [PATCH 105/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5b113d3..d9b2a92 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -49,7 +49,10 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } else { - doExport(exportfolder, exportpathlist) + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist) + } } } } From 93443e340ddb102379ccb8dc25168d6f3a8e7bc9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 12 Jun 2018 18:56:17 +0100 Subject: [PATCH 106/558] HMBXR5601-143 - Handle projects that aren't built but do provide binaries --- compile/eclipsebuild/eclipseds5build.gsh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5b113d3..d9b2a92 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -49,7 +49,10 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } else { - doExport(exportfolder, exportpathlist) + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist) + } } } } From 1c16f4b5a0cca936bc2466e2451a7ef89ad5e6e0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 25 Jun 2018 13:50:09 +0100 Subject: [PATCH 107/558] HMBXR5601-233 - Extend checkout timeout to 2 hours --- versioncontrol/git.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 70fe6bc..9d96232 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -26,7 +26,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 60], + [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] ], submoduleCfg: [], From 305ae709aacbf8cf203263211ac376f8eea2114f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 25 Jun 2018 13:50:09 +0100 Subject: [PATCH 108/558] HMBXR5601-233 - Extend checkout timeout to 2 hours --- versioncontrol/git.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 70fe6bc..9d96232 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -26,7 +26,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo extensions: [ [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], - [$class: 'CheckoutOption', timeout: 60], + [$class: 'CheckoutOption', timeout: 120], [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] ], submoduleCfg: [], From b41f1885a0dcb61754d2a5003051c54197cb0b90 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 6 Jul 2018 11:47:15 +0100 Subject: [PATCH 109/558] Add build switch to perform clean builds --- compile/eclipsebuild/eclipseds5build.gsh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index d9b2a92..5acc0ae 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -31,17 +31,23 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { // Build and export files def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, project, variants, exportpathlist) = appInfo + (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { + if (cleanbuild) { + buildtype = "-cleanBuild" + } + else { + buildtype = "-build" + } for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) + echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" if (isUnix()) { - echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + sh "${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" } else { bat """ - "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" """ } } From 8f69abb656f35569c21f193d5620057302fcd90d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 6 Jul 2018 11:47:15 +0100 Subject: [PATCH 110/558] Add build switch to perform clean builds --- compile/eclipsebuild/eclipseds5build.gsh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index d9b2a92..5acc0ae 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -31,17 +31,23 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { // Build and export files def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, project, variants, exportpathlist) = appInfo + (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { + if (cleanbuild) { + buildtype = "-cleanBuild" + } + else { + buildtype = "-build" + } for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) + echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" if (isUnix()) { - echo "Build command: ${BUILD_COMMAND} -build \"${project}/${build_variant}\"" - sh "${BUILD_COMMAND} -build \"${project}/${build_variant}\"" + sh "${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" } else { bat """ - "%BUILD_COMMAND%" -build "${project}\\${build_variant}" + "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" """ } } From ef5bcc9ffe7ade78c0eee7271e9e9dbe9eb671f4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 7 Jul 2018 19:25:33 +0100 Subject: [PATCH 111/558] Use LFS for all checkouts --- versioncontrol/git.gsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 9d96232..dbc09ff 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -24,6 +24,7 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo ], doGenerateSubmoduleConfigurations: false, extensions: [ + [$class: 'GitLFSPull'], [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], @@ -59,4 +60,4 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { return true } -return this \ No newline at end of file +return this From 746ff1edf90bd64747cc4abeb394870a3302245c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Jul 2018 19:08:38 +0100 Subject: [PATCH 112/558] HMBXR5601-328 - Use make instead of eclipsebuild --- compile/eclipsebuild/eclipseds5build.gsh | 55 ++++++++++++++++++++---- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5acc0ae..16854b7 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,37 +1,44 @@ -def doExport(exportfolder, exportpathlist, build_variant = "") { +def doExport(exportfolder, exportpathlist, build_variant = "", project_root = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" + env.srcFile = "${WORKSPACE}/${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" env.destFolder = "${exportfolder}/${build_variant}/${project}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + env.srcFile = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + } + echo "Project root is ${project_root}" + if (project_root) { + echo "Project root is ${project_root}" + env.srcFile = srcFile.replace("-PROJECTROOT-", "${project_root}") + } + else { + echo "No project root" } } dir (env.destFolder) { - echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" + echo "Copy \"${srcFile}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -v \"${srcFile}\" \"${destFolder}\"" } else { - bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' + bat 'copy \"%srcFile%\" \"%destFolder%\"' } } } } -// Build and export files +// Build and export files using Eclipse project files (unreliable) def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo + (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo if (build) { if (cleanbuild) { buildtype = "-cleanBuild" @@ -63,4 +70,34 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } +// Build and export files using make files generated by Eclipse +def doMake(apps, exportfolder = "${WORKSPACE}/bin") { + for (appInfo in apps ) { + (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo + if (build) { + for (build_variant in variants) { + echo "Changing directory to: ${WORKSPACE}/${projectroot}/${build_variant}" + dir("${WORKSPACE}/${projectroot}/${build_variant}") { + echo "Build command: ${BUILD_COMMAND}" + if (isUnix()) { + sh "${BUILD_COMMAND}" + } + else { + bat """ + "%BUILD_COMMAND%" + """ + } + } + doExport(exportfolder, exportpathlist, build_variant, projectroot) + } + } + else { + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist) + } + } + } +} + return this; From beaf754ea2eb644061322886d629740856492517 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Jul 2018 19:08:38 +0100 Subject: [PATCH 113/558] HMBXR5601-328 - Use make instead of eclipsebuild --- compile/eclipsebuild/eclipseds5build.gsh | 55 ++++++++++++++++++++---- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5acc0ae..16854b7 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,37 +1,44 @@ -def doExport(exportfolder, exportpathlist, build_variant = "") { +def doExport(exportfolder, exportpathlist, build_variant = "", project_root = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" + env.srcFile = "${WORKSPACE}/${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" env.destFolder = "${exportfolder}/${build_variant}/${project}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + env.srcFile = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + } + echo "Project root is ${project_root}" + if (project_root) { + echo "Project root is ${project_root}" + env.srcFile = srcFile.replace("-PROJECTROOT-", "${project_root}") + } + else { + echo "No project root" } } dir (env.destFolder) { - echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" + echo "Copy \"${srcFile}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -v \"${srcFile}\" \"${destFolder}\"" } else { - bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' + bat 'copy \"%srcFile%\" \"%destFolder%\"' } } } } -// Build and export files +// Build and export files using Eclipse project files (unreliable) def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo + (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo if (build) { if (cleanbuild) { buildtype = "-cleanBuild" @@ -63,4 +70,34 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } +// Build and export files using make files generated by Eclipse +def doMake(apps, exportfolder = "${WORKSPACE}/bin") { + for (appInfo in apps ) { + (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo + if (build) { + for (build_variant in variants) { + echo "Changing directory to: ${WORKSPACE}/${projectroot}/${build_variant}" + dir("${WORKSPACE}/${projectroot}/${build_variant}") { + echo "Build command: ${BUILD_COMMAND}" + if (isUnix()) { + sh "${BUILD_COMMAND}" + } + else { + bat """ + "%BUILD_COMMAND%" + """ + } + } + doExport(exportfolder, exportpathlist, build_variant, projectroot) + } + } + else { + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist) + } + } + } +} + return this; From f8da7ad8735351777617d24bd92d5caf47af28ad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Jul 2018 19:17:33 +0100 Subject: [PATCH 114/558] HMBXR5601-328 - Use make instead of eclipsebuild --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 16854b7..73c15a3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -73,7 +73,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { // Build and export files using make files generated by Eclipse def doMake(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo + (build, cleanbuild, project, variants, projectroot, exportpathlist) = appInfo if (build) { for (build_variant in variants) { echo "Changing directory to: ${WORKSPACE}/${projectroot}/${build_variant}" From c902757cc8becc57849a537b6d0e5e3216d98e4e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Jul 2018 19:17:33 +0100 Subject: [PATCH 115/558] HMBXR5601-328 - Use make instead of eclipsebuild --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 16854b7..73c15a3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -73,7 +73,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { // Build and export files using make files generated by Eclipse def doMake(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo + (build, cleanbuild, project, variants, projectroot, exportpathlist) = appInfo if (build) { for (build_variant in variants) { echo "Changing directory to: ${WORKSPACE}/${projectroot}/${build_variant}" From 3931e9b6d98671a295786267eb3ea470ca325b30 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 12:53:34 +0100 Subject: [PATCH 116/558] Revert "HMBXR5601-328 - Use make instead of eclipsebuild" This reverts commit 5e1e0c9d47cfaa4c366b84da4f8a583c93ae9693. --- compile/eclipsebuild/eclipseds5build.gsh | 55 ++++-------------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 73c15a3..5acc0ae 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,44 +1,37 @@ -def doExport(exportfolder, exportpathlist, build_variant = "", project_root = "") { +def doExport(exportfolder, exportpathlist, build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" - env.srcFile = "${WORKSPACE}/${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" env.destFolder = "${exportfolder}/${build_variant}/${project}" - env.srcFile = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" env.destFolder = "${exportfolder}/${project}" - } - echo "Project root is ${project_root}" - if (project_root) { - echo "Project root is ${project_root}" - env.srcFile = srcFile.replace("-PROJECTROOT-", "${project_root}") - } - else { - echo "No project root" + env.srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { - echo "Copy \"${srcFile}\" to \"${destFolder}\"" + echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v \"${srcFile}\" \"${destFolder}\"" + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { - bat 'copy \"%srcFile%\" \"%destFolder%\"' + bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' } } } } -// Build and export files using Eclipse project files (unreliable) +// Build and export files def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo + (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { if (cleanbuild) { buildtype = "-cleanBuild" @@ -70,34 +63,4 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } -// Build and export files using make files generated by Eclipse -def doMake(apps, exportfolder = "${WORKSPACE}/bin") { - for (appInfo in apps ) { - (build, cleanbuild, project, variants, projectroot, exportpathlist) = appInfo - if (build) { - for (build_variant in variants) { - echo "Changing directory to: ${WORKSPACE}/${projectroot}/${build_variant}" - dir("${WORKSPACE}/${projectroot}/${build_variant}") { - echo "Build command: ${BUILD_COMMAND}" - if (isUnix()) { - sh "${BUILD_COMMAND}" - } - else { - bat """ - "%BUILD_COMMAND%" - """ - } - } - doExport(exportfolder, exportpathlist, build_variant, projectroot) - } - } - else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt - if (!variants) { - doExport(exportfolder, exportpathlist) - } - } - } -} - return this; From 1869a40da9c9211225b68cc372e835149d588dad Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 12:53:34 +0100 Subject: [PATCH 117/558] Revert "HMBXR5601-328 - Use make instead of eclipsebuild" This reverts commit 5e1e0c9d47cfaa4c366b84da4f8a583c93ae9693. --- compile/eclipsebuild/eclipseds5build.gsh | 55 ++++-------------------- 1 file changed, 9 insertions(+), 46 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 73c15a3..5acc0ae 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,44 +1,37 @@ -def doExport(exportfolder, exportpathlist, build_variant = "", project_root = "") { +def doExport(exportfolder, exportpathlist, build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" - env.srcFile = "${WORKSPACE}/${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" env.destFolder = "${exportfolder}/${build_variant}/${project}" - env.srcFile = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + env.srcFile = "${WORKSPACE}/${exportpath}" + env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" env.destFolder = "${exportfolder}/${project}" - } - echo "Project root is ${project_root}" - if (project_root) { - echo "Project root is ${project_root}" - env.srcFile = srcFile.replace("-PROJECTROOT-", "${project_root}") - } - else { - echo "No project root" + env.srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { - echo "Copy \"${srcFile}\" to \"${destFolder}\"" + echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v \"${srcFile}\" \"${destFolder}\"" + sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { - bat 'copy \"%srcFile%\" \"%destFolder%\"' + bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' } } } } -// Build and export files using Eclipse project files (unreliable) +// Build and export files def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (appInfo in apps ) { - (build, cleanbuild, project, projectroot, variants, exportpathlist) = appInfo + (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { if (cleanbuild) { buildtype = "-cleanBuild" @@ -70,34 +63,4 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } -// Build and export files using make files generated by Eclipse -def doMake(apps, exportfolder = "${WORKSPACE}/bin") { - for (appInfo in apps ) { - (build, cleanbuild, project, variants, projectroot, exportpathlist) = appInfo - if (build) { - for (build_variant in variants) { - echo "Changing directory to: ${WORKSPACE}/${projectroot}/${build_variant}" - dir("${WORKSPACE}/${projectroot}/${build_variant}") { - echo "Build command: ${BUILD_COMMAND}" - if (isUnix()) { - sh "${BUILD_COMMAND}" - } - else { - bat """ - "%BUILD_COMMAND%" - """ - } - } - doExport(exportfolder, exportpathlist, build_variant, projectroot) - } - } - else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt - if (!variants) { - doExport(exportfolder, exportpathlist) - } - } - } -} - return this; From 2517f7bdc0a793b8a1a9c5671a2008725ee1e029 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 18:25:03 +0100 Subject: [PATCH 118/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5acc0ae..347f723 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -43,11 +43,14 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" if (isUnix()) { - sh "${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" + ${DS5SHELL} \'${BUILD_COMMAND} -no-indexer -importAll "${SOURCE_PATH}"\' + sh """ + ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + """ } else { bat """ - "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" + "%DS5SHELL%" "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" """ } } From aff5898578ed7fe931d7ca03a9c000630551f136 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 18:25:03 +0100 Subject: [PATCH 119/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5acc0ae..347f723 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -43,11 +43,14 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" if (isUnix()) { - sh "${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" + ${DS5SHELL} \'${BUILD_COMMAND} -no-indexer -importAll "${SOURCE_PATH}"\' + sh """ + ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + """ } else { bat """ - "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" + "%DS5SHELL%" "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" """ } } From 345a0ed28c75e9c6e9ee9e75d5b2242226a3d3d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 18:26:56 +0100 Subject: [PATCH 120/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 347f723..35ea90c 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -43,9 +43,8 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" if (isUnix()) { - ${DS5SHELL} \'${BUILD_COMMAND} -no-indexer -importAll "${SOURCE_PATH}"\' sh """ - ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' """ } else { From fbb99a88e76ff545e81e5957e6d9f7f507de7bb7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 18:26:56 +0100 Subject: [PATCH 121/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 347f723..35ea90c 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -43,9 +43,8 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" if (isUnix()) { - ${DS5SHELL} \'${BUILD_COMMAND} -no-indexer -importAll "${SOURCE_PATH}"\' sh """ - ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' """ } else { From 9836cd869023d4f65097ce6958ac6f919277bff6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 18:37:18 +0100 Subject: [PATCH 122/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 35ea90c..bbd17af 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -41,7 +41,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) - echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" + echo "Build command: \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' From 8005e0d3c6709b67499fd4c43bb41f1a25ac837a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 18:37:18 +0100 Subject: [PATCH 123/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 35ea90c..bbd17af 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -41,7 +41,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) - echo "Build command: ${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"" + echo "Build command: \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' From 8141c493b1d44f92035cb4b8a420e5c37f918593 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 19:24:34 +0100 Subject: [PATCH 124/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index bbd17af..6c121e3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -41,10 +41,10 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) - echo "Build command: \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" + echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ - \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' """ } else { From 804a86aeaababbacdab5cbab7484f40891404ec4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 19:24:34 +0100 Subject: [PATCH 125/558] Seperate DS5 shell command from build command --- compile/eclipsebuild/eclipseds5build.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index bbd17af..6c121e3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -41,10 +41,10 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } for (build_variant in variants) { retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) - echo "Build command: \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" + echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ - \"${DS5SHELL}\" \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' """ } else { From a4a3c387c9695e268999da04157d60394212759e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 23:27:58 +0100 Subject: [PATCH 126/558] Parameterise eclipse project build retries --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 6c121e3..b5f4e16 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -40,7 +40,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { buildtype = "-build" } for (build_variant in variants) { - retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ From fd7b0486cf076532cf5fec95b326c321d2ea2ce0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Jul 2018 23:27:58 +0100 Subject: [PATCH 127/558] Parameterise eclipse project build retries --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 6c121e3..b5f4e16 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -40,7 +40,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { buildtype = "-build" } for (build_variant in variants) { - retry(3) { // Eclipse occasionally raises exceptions (rather than the builds failing) + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ From 628f750659481bea49d5fcc88981da30add31b68 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 22 Aug 2018 13:50:34 +0100 Subject: [PATCH 128/558] HMBXR5601-390 - Copy folders recursively --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index b5f4e16..38cbb40 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -19,7 +19,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh "cp -r -v \"${srcFileExpanded}\" \"${destFolder}\"" } else { bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' From 39e8844bde13d31f1310a62f6223a26bef0cc34b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Sep 2018 16:14:04 +0100 Subject: [PATCH 129/558] Cause a project build retry if the final binary was not produced --- compile/eclipsebuild/eclipseds5build.gsh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 38cbb40..0ff36da 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -19,10 +19,16 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -r -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh """ + cp -r -v "${srcFileExpanded}" "${destFolder}" + if [ \$? -ne 0 ]; then exit 1; fi + """ } else { - bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' + bat ''' + copy \"%srcFileExpanded%\" \"%destFolder%\" + if errorlevel 1 exit /b 1 + ''' } } } From f9e2359888cfa2acec7097a4ccbbba3574e788a9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Sep 2018 16:14:04 +0100 Subject: [PATCH 130/558] Cause a project build retry if the final binary was not produced --- compile/eclipsebuild/eclipseds5build.gsh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 38cbb40..0ff36da 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -19,10 +19,16 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { dir (env.destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { - sh "cp -r -v \"${srcFileExpanded}\" \"${destFolder}\"" + sh """ + cp -r -v "${srcFileExpanded}" "${destFolder}" + if [ \$? -ne 0 ]; then exit 1; fi + """ } else { - bat 'copy \"%srcFileExpanded%\" \"%destFolder%\"' + bat ''' + copy \"%srcFileExpanded%\" \"%destFolder%\" + if errorlevel 1 exit /b 1 + ''' } } } From aec43dde0a08b9237e0abc6cc5f4f6597cb7c96a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Sep 2018 16:58:30 +0100 Subject: [PATCH 131/558] Look out for HMB tickets when comparing revision differences --- differences/show_release_differences.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index a778892..93d540b 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -72,6 +72,12 @@ jiraprojectpattern = { 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From d77a2dbcc8c215680d71470606981a69b46068f2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Sep 2018 16:58:30 +0100 Subject: [PATCH 132/558] Look out for HMB tickets when comparing revision differences --- differences/show_release_differences.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index a778892..93d540b 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -72,6 +72,12 @@ jiraprojectpattern = { 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From 3043d69d993bd00381300502efbe7d9cef62a197 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Sep 2018 18:48:54 +0100 Subject: [PATCH 133/558] Disable parallel builds; Retry build if copy fails --- .../eclipsebuild/disableparallelbuilds.gsh | 12 ++++++++++ compile/eclipsebuild/eclipseds5build.gsh | 24 +++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 compile/eclipsebuild/disableparallelbuilds.gsh diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh new file mode 100644 index 0000000..77f5eb5 --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " Date: Sat, 8 Sep 2018 18:48:54 +0100 Subject: [PATCH 134/558] Disable parallel builds; Retry build if copy fails --- .../eclipsebuild/disableparallelbuilds.gsh | 12 ++++++++++ compile/eclipsebuild/eclipseds5build.gsh | 24 +++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 compile/eclipsebuild/disableparallelbuilds.gsh diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh new file mode 100644 index 0000000..77f5eb5 --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " Date: Sat, 8 Sep 2018 18:53:43 +0100 Subject: [PATCH 135/558] Disable parallel builds; Retry build if copy fails --- compile/eclipsebuild/disableparallelbuilds.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh index 77f5eb5..aeca69a 100644 --- a/compile/eclipsebuild/disableparallelbuilds.gsh +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -4,7 +4,7 @@ def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { if (isUnix()) { sh """ # Looks for " Date: Sat, 8 Sep 2018 18:53:43 +0100 Subject: [PATCH 136/558] Disable parallel builds; Retry build if copy fails --- compile/eclipsebuild/disableparallelbuilds.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh index 77f5eb5..aeca69a 100644 --- a/compile/eclipsebuild/disableparallelbuilds.gsh +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -4,7 +4,7 @@ def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { if (isUnix()) { sh """ # Looks for " Date: Sat, 8 Sep 2018 18:57:09 +0100 Subject: [PATCH 137/558] Disable parallel builds; Retry build if copy fails --- compile/eclipsebuild/disableparallelbuilds.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh index aeca69a..c4322cd 100644 --- a/compile/eclipsebuild/disableparallelbuilds.gsh +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -4,7 +4,7 @@ def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { if (isUnix()) { sh """ # Looks for " Date: Sat, 8 Sep 2018 18:57:09 +0100 Subject: [PATCH 138/558] Disable parallel builds; Retry build if copy fails --- compile/eclipsebuild/disableparallelbuilds.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh index aeca69a..c4322cd 100644 --- a/compile/eclipsebuild/disableparallelbuilds.gsh +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -4,7 +4,7 @@ def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { if (isUnix()) { sh """ # Looks for " Date: Sat, 8 Sep 2018 18:59:30 +0100 Subject: [PATCH 139/558] Disable parallel builds; Retry build if copy fails --- compile/eclipsebuild/disableparallelbuilds.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh index c4322cd..03e4c8f 100644 --- a/compile/eclipsebuild/disableparallelbuilds.gsh +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -4,7 +4,7 @@ def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { if (isUnix()) { sh """ # Looks for " Date: Sat, 8 Sep 2018 18:59:30 +0100 Subject: [PATCH 140/558] Disable parallel builds; Retry build if copy fails --- compile/eclipsebuild/disableparallelbuilds.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.gsh index c4322cd..03e4c8f 100644 --- a/compile/eclipsebuild/disableparallelbuilds.gsh +++ b/compile/eclipsebuild/disableparallelbuilds.gsh @@ -4,7 +4,7 @@ def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { if (isUnix()) { sh """ # Looks for " Date: Sat, 8 Sep 2018 19:53:12 +0100 Subject: [PATCH 141/558] Put all compiled binaries in one variant specific folder on the target - making it easier to find all shared objects --- compile/eclipsebuild/eclipseds5build.gsh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5d8c5a3..9f702fe 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,5 +1,5 @@ -def doExport(exportfolder, exportpathlist, build_variant = "") { +def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" @@ -7,7 +7,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}/${project}" + env.destFolder = "${exportfolder}/${build_variant}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } @@ -68,14 +68,14 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { "%DS5SHELL%" "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" """ } - doExport(exportfolder, exportpathlist, build_variant) + doExport(exportfolder, exportpathlist, "", build_variant) } } } else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - doExport(exportfolder, exportpathlist) + doExport(exportfolder, exportpathlist, project) } } } From 128f4b117a93dda82105d35be16f3de9b8dbfbb4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 8 Sep 2018 19:53:12 +0100 Subject: [PATCH 142/558] Put all compiled binaries in one variant specific folder on the target - making it easier to find all shared objects --- compile/eclipsebuild/eclipseds5build.gsh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5d8c5a3..9f702fe 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,5 +1,5 @@ -def doExport(exportfolder, exportpathlist, build_variant = "") { +def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" @@ -7,7 +7,7 @@ def doExport(exportfolder, exportpathlist, build_variant = "") { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}/${project}" + env.destFolder = "${exportfolder}/${build_variant}" env.srcFile = "${WORKSPACE}/${exportpath}" env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } @@ -68,14 +68,14 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { "%DS5SHELL%" "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" """ } - doExport(exportfolder, exportpathlist, build_variant) + doExport(exportfolder, exportpathlist, "", build_variant) } } } else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - doExport(exportfolder, exportpathlist) + doExport(exportfolder, exportpathlist, project) } } } From 509367c0574d64ed3d8cb2a8a698a40eff8b1284 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Sep 2018 14:17:42 +0100 Subject: [PATCH 143/558] Record time to build a project --- compile/eclipsebuild/eclipseds5build.gsh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9f702fe..dfefd02 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -60,7 +60,9 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ + echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` """ } else { From b6f9a430bcd61cf537265bf94f996a864dfc91ef Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Sep 2018 14:17:42 +0100 Subject: [PATCH 144/558] Record time to build a project --- compile/eclipsebuild/eclipseds5build.gsh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9f702fe..dfefd02 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -60,7 +60,9 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ + echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` """ } else { From e9e01af18712ab524f711455811d1cc588d0cbd5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 16:34:03 +0100 Subject: [PATCH 145/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index dfefd02..1c213d5 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,4 +1,19 @@ +def doKillLeftovers() { + if (isUnix()) { + sh """ + # Killing any processes that may interfere with the next build + # The build system must be setup to do more than one eclipse build concurrently + ps aux | grep "eclipsews" + ps aux | grep "make" + ps aux | grep "arm-linux" + pkill -9 eclipsews + pkill -9 make + pkill -9 "*arm-linux*" + """ + } +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 for (exportpath in exportpathlist) { @@ -60,6 +75,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ + doKillLeftovers() echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` From e466892cde2edb19297f14075f45ba960e9122e2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 16:34:03 +0100 Subject: [PATCH 146/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index dfefd02..1c213d5 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,4 +1,19 @@ +def doKillLeftovers() { + if (isUnix()) { + sh """ + # Killing any processes that may interfere with the next build + # The build system must be setup to do more than one eclipse build concurrently + ps aux | grep "eclipsews" + ps aux | grep "make" + ps aux | grep "arm-linux" + pkill -9 eclipsews + pkill -9 make + pkill -9 "*arm-linux*" + """ + } +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 for (exportpath in exportpathlist) { @@ -60,6 +75,7 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ + doKillLeftovers() echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` From aa25d4d5614f56e3b2055aee3d1ddaf2c18ebe5c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 16:44:21 +0100 Subject: [PATCH 147/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 1c213d5..68bb64a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -73,9 +73,9 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" + doKillLeftovers() if (isUnix()) { sh """ - doKillLeftovers() echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` From bffdde6145cce055cf05916eefec16d92bf32c5e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 16:44:21 +0100 Subject: [PATCH 148/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 1c213d5..68bb64a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -73,9 +73,9 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" + doKillLeftovers() if (isUnix()) { sh """ - doKillLeftovers() echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` From 0bb2733aae93427abbdc8c05027ad7592b67fbbd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 17:10:57 +0100 Subject: [PATCH 149/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 68bb64a..6cc4c31 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -7,9 +7,10 @@ def doKillLeftovers() { ps aux | grep "eclipsews" ps aux | grep "make" ps aux | grep "arm-linux" - pkill -9 eclipsews - pkill -9 make - pkill -9 "*arm-linux*" + # Don't return an error if the processes don't exist + pkill -9 eclipsews || true + pkill -9 make || true + pkill -9 "*arm-linux*" || true """ } } From a7eb5454e75ed47b3b8e52a353f180a136840555 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 17:10:57 +0100 Subject: [PATCH 150/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 68bb64a..6cc4c31 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -7,9 +7,10 @@ def doKillLeftovers() { ps aux | grep "eclipsews" ps aux | grep "make" ps aux | grep "arm-linux" - pkill -9 eclipsews - pkill -9 make - pkill -9 "*arm-linux*" + # Don't return an error if the processes don't exist + pkill -9 eclipsews || true + pkill -9 make || true + pkill -9 "*arm-linux*" || true """ } } From 22605b6677af1c8bb227222d0ffd19905b9e7bcc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 17:16:19 +0100 Subject: [PATCH 151/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 6cc4c31..c7804fa 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -10,7 +10,7 @@ def doKillLeftovers() { # Don't return an error if the processes don't exist pkill -9 eclipsews || true pkill -9 make || true - pkill -9 "*arm-linux*" || true + pkill -9 ".*arm-linux.*" || true """ } } From 77202e4488e592b1bce6ae134902235e1cb8e917 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 17:16:19 +0100 Subject: [PATCH 152/558] Kill any left overs from previous builds before proceeding --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 6cc4c31..c7804fa 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -10,7 +10,7 @@ def doKillLeftovers() { # Don't return an error if the processes don't exist pkill -9 eclipsews || true pkill -9 make || true - pkill -9 "*arm-linux*" || true + pkill -9 ".*arm-linux.*" || true """ } } From 5d45c483cc479f331d6edf16e52d7f01dba425e9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 19:31:17 +0100 Subject: [PATCH 153/558] Register toolchain multiple times --- compile/eclipsebuild/eclipseds5build.gsh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index c7804fa..2be3d95 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -77,6 +77,8 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() if (isUnix()) { sh """ + # Register toolchain (if it is not already) + "${DS5_FOLDER}/bin/add_toolchain" --non-interactive "${TOOLCHAIN_FOLDER}" echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` From 99d341a1b0fe9ee803594b0c622f7bc3cd5452e3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 14 Sep 2018 19:31:17 +0100 Subject: [PATCH 154/558] Register toolchain multiple times --- compile/eclipsebuild/eclipseds5build.gsh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index c7804fa..2be3d95 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -77,6 +77,8 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() if (isUnix()) { sh """ + # Register toolchain (if it is not already) + "${DS5_FOLDER}/bin/add_toolchain" --non-interactive "${TOOLCHAIN_FOLDER}" echo Start time: `date` ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` From e5aa2bc6407133590cc4ee57e254b943288a2452 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 12:52:17 +0100 Subject: [PATCH 155/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 33 ++++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index c0e28c0..1095d32 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -61,7 +61,27 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } // Build and export files -def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { +def doRegisterToolchain(ds5folder, toolchainfolder) { + # Register toolchain (if it is not already) + if (isUnix()) { + sh """ + "${ds5folder}/bin/add_toolchain" --non-interactive "${toolchainfolder}" + """ + env.PATH = env.TOOLCHAIN_FOLDER + ":" + env.PATH + } + else { + bat """ + "%ds5folder%\\bin\\add_toolchain" --non-interactive "%toolchainfolder%" + """ + env.PATH = env.TOOLCHAIN_FOLDER + ";" + env.PATH + } + echo "Path is: ${PATH}" +} + +// Build and export files +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { + doKillLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { @@ -73,20 +93,17 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" - doKillLeftovers() + echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ - # Register toolchain (if it is not already) - "${DS5_FOLDER}/bin/add_toolchain" --non-interactive "${TOOLCHAIN_FOLDER}" echo Start time: `date` - ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` """ } else { bat """ - "%DS5SHELL%" "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" + "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" """ } doExport(exportfolder, exportpathlist, "", build_variant) @@ -102,4 +119,4 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } -return this; \ No newline at end of file +return this; From 05c18c11a45d9765fb616956f278ac69e5a7ab97 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 12:52:17 +0100 Subject: [PATCH 156/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 33 ++++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index c0e28c0..1095d32 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -61,7 +61,27 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } // Build and export files -def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { +def doRegisterToolchain(ds5folder, toolchainfolder) { + # Register toolchain (if it is not already) + if (isUnix()) { + sh """ + "${ds5folder}/bin/add_toolchain" --non-interactive "${toolchainfolder}" + """ + env.PATH = env.TOOLCHAIN_FOLDER + ":" + env.PATH + } + else { + bat """ + "%ds5folder%\\bin\\add_toolchain" --non-interactive "%toolchainfolder%" + """ + env.PATH = env.TOOLCHAIN_FOLDER + ";" + env.PATH + } + echo "Path is: ${PATH}" +} + +// Build and export files +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { + doKillLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { @@ -73,20 +93,17 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - echo "Build command: ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\'" - doKillLeftovers() + echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ - # Register toolchain (if it is not already) - "${DS5_FOLDER}/bin/add_toolchain" --non-interactive "${TOOLCHAIN_FOLDER}" echo Start time: `date` - ${DS5SHELL} \'${BUILD_COMMAND} ${buildtype} \"${project}/${build_variant}\"\' + ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` """ } else { bat """ - "%DS5SHELL%" "%BUILD_COMMAND%" ${buildtype} "${project}\\${build_variant}" + "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" """ } doExport(exportfolder, exportpathlist, "", build_variant) @@ -102,4 +119,4 @@ def doBuild(apps, exportfolder = "${WORKSPACE}/bin") { } } -return this; \ No newline at end of file +return this; From d2067da0546976829b1eb73e4e0d4d6274b02352 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 13:10:12 +0100 Subject: [PATCH 157/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 1095d32..8d781a4 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -62,7 +62,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { // Build and export files def doRegisterToolchain(ds5folder, toolchainfolder) { - # Register toolchain (if it is not already) + // Register toolchain (if it is not already) if (isUnix()) { sh """ "${ds5folder}/bin/add_toolchain" --non-interactive "${toolchainfolder}" From 8783c9a9c3b960488de81a67fdcbb8e37157665b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 13:10:12 +0100 Subject: [PATCH 158/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 1095d32..8d781a4 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -62,7 +62,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { // Build and export files def doRegisterToolchain(ds5folder, toolchainfolder) { - # Register toolchain (if it is not already) + // Register toolchain (if it is not already) if (isUnix()) { sh """ "${ds5folder}/bin/add_toolchain" --non-interactive "${toolchainfolder}" From cfa2b57ec79b6a111ba84b94f95f9e1c2eaf8710 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 13:15:17 +0100 Subject: [PATCH 159/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 8d781a4..7409d96 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -81,7 +81,7 @@ def doRegisterToolchain(ds5folder, toolchainfolder) { // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder + doRegisterToolchain(ds5folder, toolchainfolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { From c72196569f5c5d563095137f4724d0bdf6ddcc0d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 13:15:17 +0100 Subject: [PATCH 160/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 8d781a4..7409d96 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -81,7 +81,7 @@ def doRegisterToolchain(ds5folder, toolchainfolder) { // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder + doRegisterToolchain(ds5folder, toolchainfolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { From 78fbf0a9a7ad8699794816af84e745f4190bf62e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 13:42:44 +0100 Subject: [PATCH 161/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 7409d96..69e9884 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -60,20 +60,30 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { return result } -// Build and export files +// Register toolchain def doRegisterToolchain(ds5folder, toolchainfolder) { // Register toolchain (if it is not already) if (isUnix()) { sh """ "${ds5folder}/bin/add_toolchain" --non-interactive "${toolchainfolder}" """ - env.PATH = env.TOOLCHAIN_FOLDER + ":" + env.PATH } else { bat """ "%ds5folder%\\bin\\add_toolchain" --non-interactive "%toolchainfolder%" """ - env.PATH = env.TOOLCHAIN_FOLDER + ";" + env.PATH + } + echo "Path is: ${PATH}" +} + +// Add toolchain to path +def addToolchainPath(toolchainfolder) { + // Register toolchain (if it is not already) + if (isUnix()) { + env.PATH = toolchainfolder + ":" + env.PATH + } + else { + env.PATH = toolchainfolder + ";" + env.PATH } echo "Path is: ${PATH}" } @@ -81,7 +91,7 @@ def doRegisterToolchain(ds5folder, toolchainfolder) { // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) + addToolchainPath(toolchainfolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { @@ -93,6 +103,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exp } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ From cd889893dd577a529f1f64e927815b888f1c3ec7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 13:42:44 +0100 Subject: [PATCH 162/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 7409d96..69e9884 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -60,20 +60,30 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { return result } -// Build and export files +// Register toolchain def doRegisterToolchain(ds5folder, toolchainfolder) { // Register toolchain (if it is not already) if (isUnix()) { sh """ "${ds5folder}/bin/add_toolchain" --non-interactive "${toolchainfolder}" """ - env.PATH = env.TOOLCHAIN_FOLDER + ":" + env.PATH } else { bat """ "%ds5folder%\\bin\\add_toolchain" --non-interactive "%toolchainfolder%" """ - env.PATH = env.TOOLCHAIN_FOLDER + ";" + env.PATH + } + echo "Path is: ${PATH}" +} + +// Add toolchain to path +def addToolchainPath(toolchainfolder) { + // Register toolchain (if it is not already) + if (isUnix()) { + env.PATH = toolchainfolder + ":" + env.PATH + } + else { + env.PATH = toolchainfolder + ";" + env.PATH } echo "Path is: ${PATH}" } @@ -81,7 +91,7 @@ def doRegisterToolchain(ds5folder, toolchainfolder) { // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) + addToolchainPath(toolchainfolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { @@ -93,6 +103,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exp } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ From 1eced1a5a4947f65436104da2ac7a1c56c43f4c7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 14:19:21 +0100 Subject: [PATCH 163/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 69e9884..430cfca 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -88,10 +88,24 @@ def addToolchainPath(toolchainfolder) { echo "Path is: ${PATH}" } +// Import all eclipse projects in a given folder +def doImportProjects(ds5shellcommand, buildcommand, sourcefolder) { + doRegisterToolchain(ds5folder, toolchainfolder) + if (isUnix()) { + sh """ + ${ds5shellcommand} \'${buildcommand} -no-indexer -importAll "${sourcefolder}"\' + """ + } + else { + bat '%ds5shellcommand% %buildcommand% -no-indexer -importAll "%sourcefolder%"' + } +} + // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() addToolchainPath(toolchainfolder) + doImportProjects(ds5shellcommand, buildcommand, sourcefolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { From c2425b61656405cc4f585d67ca301b642b068ccc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 14:19:21 +0100 Subject: [PATCH 164/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 69e9884..430cfca 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -88,10 +88,24 @@ def addToolchainPath(toolchainfolder) { echo "Path is: ${PATH}" } +// Import all eclipse projects in a given folder +def doImportProjects(ds5shellcommand, buildcommand, sourcefolder) { + doRegisterToolchain(ds5folder, toolchainfolder) + if (isUnix()) { + sh """ + ${ds5shellcommand} \'${buildcommand} -no-indexer -importAll "${sourcefolder}"\' + """ + } + else { + bat '%ds5shellcommand% %buildcommand% -no-indexer -importAll "%sourcefolder%"' + } +} + // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() addToolchainPath(toolchainfolder) + doImportProjects(ds5shellcommand, buildcommand, sourcefolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { From fc72420338ce08944b6b7493b1ea7a9c27057c1e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 14:28:30 +0100 Subject: [PATCH 165/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 430cfca..f05603a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -89,7 +89,7 @@ def addToolchainPath(toolchainfolder) { } // Import all eclipse projects in a given folder -def doImportProjects(ds5shellcommand, buildcommand, sourcefolder) { +def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) { doRegisterToolchain(ds5folder, toolchainfolder) if (isUnix()) { sh """ @@ -105,7 +105,7 @@ def doImportProjects(ds5shellcommand, buildcommand, sourcefolder) { def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() addToolchainPath(toolchainfolder) - doImportProjects(ds5shellcommand, buildcommand, sourcefolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { From 749a7baa710f01866fb1b8f6189be0223c2a3e9e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Sep 2018 14:28:30 +0100 Subject: [PATCH 166/558] Now that the build has become reliable - remove some debug; Refactor clean up and build set up --- compile/eclipsebuild/eclipseds5build.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 430cfca..f05603a 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -89,7 +89,7 @@ def addToolchainPath(toolchainfolder) { } // Import all eclipse projects in a given folder -def doImportProjects(ds5shellcommand, buildcommand, sourcefolder) { +def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) { doRegisterToolchain(ds5folder, toolchainfolder) if (isUnix()) { sh """ @@ -105,7 +105,7 @@ def doImportProjects(ds5shellcommand, buildcommand, sourcefolder) { def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { doKillLeftovers() addToolchainPath(toolchainfolder) - doImportProjects(ds5shellcommand, buildcommand, sourcefolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { From 2a48022f4638065fc7c813890d19f15f858495cd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Sep 2018 23:24:25 +0100 Subject: [PATCH 167/558] Add ability to build device tree from source as opposed to deriving from sopcinfo file --- compile/eclipsebuild/eclipseds5build.gsh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index f05603a..2a6fce3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -4,13 +4,16 @@ def doKillLeftovers() { sh """ # Killing any processes that may interfere with the next build # The build system must be setup to do more than one eclipse build concurrently - ps aux | grep "eclipsews" - ps aux | grep "make" - ps aux | grep "arm-linux" + USER=`whoami` + pgrep -U "${USER}" "eclipse" + pgrep -U "${USER}" "eclipsews" + pgrep -U "${USER}" "make" + pgrep -U "${USER}" "arm-linux" # Don't return an error if the processes don't exist - pkill -9 eclipsews || true - pkill -9 make || true - pkill -9 ".*arm-linux.*" || true + pkill -U "${USER}" -9 eclipse || true + pkill -U "${USER}" -9 eclipsews || true + pkill -U "${USER}" -9 make || true + pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } } From 460081435558725db1144a407142865bf237fa23 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Sep 2018 23:24:25 +0100 Subject: [PATCH 168/558] Add ability to build device tree from source as opposed to deriving from sopcinfo file --- compile/eclipsebuild/eclipseds5build.gsh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index f05603a..2a6fce3 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -4,13 +4,16 @@ def doKillLeftovers() { sh """ # Killing any processes that may interfere with the next build # The build system must be setup to do more than one eclipse build concurrently - ps aux | grep "eclipsews" - ps aux | grep "make" - ps aux | grep "arm-linux" + USER=`whoami` + pgrep -U "${USER}" "eclipse" + pgrep -U "${USER}" "eclipsews" + pgrep -U "${USER}" "make" + pgrep -U "${USER}" "arm-linux" # Don't return an error if the processes don't exist - pkill -9 eclipsews || true - pkill -9 make || true - pkill -9 ".*arm-linux.*" || true + pkill -U "${USER}" -9 eclipse || true + pkill -U "${USER}" -9 eclipsews || true + pkill -U "${USER}" -9 make || true + pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } } From 7c682da874bd8ac0f50256e04cbb7e6763f98e9f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 00:22:21 +0100 Subject: [PATCH 169/558] Correct pkill call --- compile/eclipsebuild/eclipseds5build.gsh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 2a6fce3..731c6b4 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -5,10 +5,10 @@ def doKillLeftovers() { # Killing any processes that may interfere with the next build # The build system must be setup to do more than one eclipse build concurrently USER=`whoami` - pgrep -U "${USER}" "eclipse" - pgrep -U "${USER}" "eclipsews" - pgrep -U "${USER}" "make" - pgrep -U "${USER}" "arm-linux" + pgrep -U "${USER}" "eclipse" || true + pgrep -U "${USER}" "eclipsews" || true + pgrep -U "${USER}" "make" || true + pgrep -U "${USER}" "arm-linux" || true # Don't return an error if the processes don't exist pkill -U "${USER}" -9 eclipse || true pkill -U "${USER}" -9 eclipsews || true From 36dffd80e636a43d25eb0ab7c2a76111a9f0d3c1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 00:22:21 +0100 Subject: [PATCH 170/558] Correct pkill call --- compile/eclipsebuild/eclipseds5build.gsh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 2a6fce3..731c6b4 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -5,10 +5,10 @@ def doKillLeftovers() { # Killing any processes that may interfere with the next build # The build system must be setup to do more than one eclipse build concurrently USER=`whoami` - pgrep -U "${USER}" "eclipse" - pgrep -U "${USER}" "eclipsews" - pgrep -U "${USER}" "make" - pgrep -U "${USER}" "arm-linux" + pgrep -U "${USER}" "eclipse" || true + pgrep -U "${USER}" "eclipsews" || true + pgrep -U "${USER}" "make" || true + pgrep -U "${USER}" "arm-linux" || true # Don't return an error if the processes don't exist pkill -U "${USER}" -9 eclipse || true pkill -U "${USER}" -9 eclipsews || true From 953e3dfc0051dcdb800e4b5c3249629ca92f1fd3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 09:08:28 +0100 Subject: [PATCH 171/558] Stop builds killing each other on the same slave --- compile/eclipsebuild/eclipseds5build.gsh | 79 ++++++++++++------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 731c6b4..4638179 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -4,15 +4,13 @@ def doKillLeftovers() { sh """ # Killing any processes that may interfere with the next build # The build system must be setup to do more than one eclipse build concurrently - USER=`whoami` - pgrep -U "${USER}" "eclipse" || true - pgrep -U "${USER}" "eclipsews" || true - pgrep -U "${USER}" "make" || true - pgrep -U "${USER}" "arm-linux" || true # Don't return an error if the processes don't exist + USER=`whoami` + ps aux | grep "eclipse" pkill -U "${USER}" -9 eclipse || true - pkill -U "${USER}" -9 eclipsews || true + ps aux | grep "make" pkill -U "${USER}" -9 make || true + ps aux | grep "arm-linux" pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } @@ -106,42 +104,45 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { - doKillLeftovers() - addToolchainPath(toolchainfolder) - doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) - for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo - if (build) { - if (cleanbuild) { - buildtype = "-cleanBuild" - } - else { - buildtype = "-build" - } - for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + // The lock is being used because it was being found that the doKillLeftovers was affecting other builds + lock(resource: "eclipsebuild_${env.NODE_NAME}") { + doKillLeftovers() + addToolchainPath(toolchainfolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + for (appInfo in apps ) { + (build, cleanbuild, project, variants, exportpathlist) = appInfo + if (build) { + if (cleanbuild) { + buildtype = "-cleanBuild" + } + else { + buildtype = "-build" + } + for (build_variant in variants) { + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" + """ + } + doExport(exportfolder, exportpathlist, "", build_variant) } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" - """ - } - doExport(exportfolder, exportpathlist, "", build_variant) } } - } - else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt - if (!variants) { - doExport(exportfolder, exportpathlist, project) + else { + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist, project) + } } } } From dc977fe2558012f89c56fa46aa66935ad68d1814 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 09:08:28 +0100 Subject: [PATCH 172/558] Stop builds killing each other on the same slave --- compile/eclipsebuild/eclipseds5build.gsh | 79 ++++++++++++------------ 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 731c6b4..4638179 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -4,15 +4,13 @@ def doKillLeftovers() { sh """ # Killing any processes that may interfere with the next build # The build system must be setup to do more than one eclipse build concurrently - USER=`whoami` - pgrep -U "${USER}" "eclipse" || true - pgrep -U "${USER}" "eclipsews" || true - pgrep -U "${USER}" "make" || true - pgrep -U "${USER}" "arm-linux" || true # Don't return an error if the processes don't exist + USER=`whoami` + ps aux | grep "eclipse" pkill -U "${USER}" -9 eclipse || true - pkill -U "${USER}" -9 eclipsews || true + ps aux | grep "make" pkill -U "${USER}" -9 make || true + ps aux | grep "arm-linux" pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } @@ -106,42 +104,45 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { - doKillLeftovers() - addToolchainPath(toolchainfolder) - doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) - for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo - if (build) { - if (cleanbuild) { - buildtype = "-cleanBuild" - } - else { - buildtype = "-build" - } - for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + // The lock is being used because it was being found that the doKillLeftovers was affecting other builds + lock(resource: "eclipsebuild_${env.NODE_NAME}") { + doKillLeftovers() + addToolchainPath(toolchainfolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + for (appInfo in apps ) { + (build, cleanbuild, project, variants, exportpathlist) = appInfo + if (build) { + if (cleanbuild) { + buildtype = "-cleanBuild" + } + else { + buildtype = "-build" + } + for (build_variant in variants) { + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" + """ + } + doExport(exportfolder, exportpathlist, "", build_variant) } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" - """ - } - doExport(exportfolder, exportpathlist, "", build_variant) } } - } - else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt - if (!variants) { - doExport(exportfolder, exportpathlist, project) + else { + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist, project) + } } } } From a2b0e937055c4ed1b10c11d3047c9fba3d87e0f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 09:21:27 +0100 Subject: [PATCH 173/558] Stop builds killing each other on the same slave --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 4638179..9cf1c57 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -7,11 +7,11 @@ def doKillLeftovers() { # Don't return an error if the processes don't exist USER=`whoami` ps aux | grep "eclipse" - pkill -U "${USER}" -9 eclipse || true + # pkill -U "${USER}" -9 eclipse || true ps aux | grep "make" - pkill -U "${USER}" -9 make || true + # pkill -U "${USER}" -9 make || true ps aux | grep "arm-linux" - pkill -U "${USER}" -9 ".*arm-linux.*" || true + # pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } } From fc8bf281ae04c7888a7fbaa25f7fed9c5a83aa5c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 09:21:27 +0100 Subject: [PATCH 174/558] Stop builds killing each other on the same slave --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 4638179..9cf1c57 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -7,11 +7,11 @@ def doKillLeftovers() { # Don't return an error if the processes don't exist USER=`whoami` ps aux | grep "eclipse" - pkill -U "${USER}" -9 eclipse || true + # pkill -U "${USER}" -9 eclipse || true ps aux | grep "make" - pkill -U "${USER}" -9 make || true + # pkill -U "${USER}" -9 make || true ps aux | grep "arm-linux" - pkill -U "${USER}" -9 ".*arm-linux.*" || true + # pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } } From 6b10e598777e98f6521ae7c57ebcbf9674ddb895 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 13:35:39 +0100 Subject: [PATCH 175/558] Move -no-indexer parameter into a consistent position --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9cf1c57..7de42dd 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -121,17 +121,17 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` """ } else { bat """ - "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" """ } doExport(exportfolder, exportpathlist, "", build_variant) From 35cab07aff5d1e5a99acd4b715ef4e7db6e6392d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 13:35:39 +0100 Subject: [PATCH 176/558] Move -no-indexer parameter into a consistent position --- compile/eclipsebuild/eclipseds5build.gsh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9cf1c57..7de42dd 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -121,17 +121,17 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\'" + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { sh """ echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} ${buildtype} \"${project}/${build_variant}\"\' + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` """ } else { bat """ - "%ds5shellcommand%" "%buildcommand%" ${buildtype} "${project}\\${build_variant}" + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" """ } doExport(exportfolder, exportpathlist, "", build_variant) From acc8281b25da88d2ba942ce16bf425bc5fe63573 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 13:45:37 +0100 Subject: [PATCH 177/558] Kill leftovers after a successful or failed build --- compile/eclipsebuild/eclipseds5build.gsh | 93 ++++++++++++++---------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 7de42dd..3444d25 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,5 +1,4 @@ - -def doKillLeftovers() { +def doDisplayLeftovers() { if (isUnix()) { sh """ # Killing any processes that may interfere with the next build @@ -7,11 +6,23 @@ def doKillLeftovers() { # Don't return an error if the processes don't exist USER=`whoami` ps aux | grep "eclipse" - # pkill -U "${USER}" -9 eclipse || true ps aux | grep "make" - # pkill -U "${USER}" -9 make || true ps aux | grep "arm-linux" - # pkill -U "${USER}" -9 ".*arm-linux.*" || true + """ + } +} + +def doKillLeftovers() { + if (isUnix()) { + doDisplayLeftovers() + sh """ + # Killing any processes that may interfere with the next build + # The build system must be setup to do more than one eclipse build concurrently + # Don't return an error if the processes don't exist + USER=`whoami` + pkill -U "${USER}" -9 eclipse || true + pkill -U "${USER}" -9 make || true + pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } } @@ -106,45 +117,51 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { - doKillLeftovers() - addToolchainPath(toolchainfolder) - doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) - for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo - if (build) { - if (cleanbuild) { - buildtype = "-cleanBuild" + doDisplayLeftovers() + try { + addToolchainPath(toolchainfolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + for (appInfo in apps ) { + (build, cleanbuild, project, variants, exportpathlist) = appInfo + if (build) { + if (cleanbuild) { + buildtype = "-cleanBuild" + } + else { + buildtype = "-build" + } + for (build_variant in variants) { + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } + doExport(exportfolder, exportpathlist, "", build_variant) + } + } } else { - buildtype = "-build" - } - for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } - doExport(exportfolder, exportpathlist, "", build_variant) + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist, project) } } } - else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt - if (!variants) { - doExport(exportfolder, exportpathlist, project) - } - } + } catch (Exception e) { + error "Build failed - Killing leftovers" + doKillLeftovers() } + doKillLeftovers() } } From c3b1af3b93893dc7cc1e110ccd7cf371830cae56 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 13:45:37 +0100 Subject: [PATCH 178/558] Kill leftovers after a successful or failed build --- compile/eclipsebuild/eclipseds5build.gsh | 93 ++++++++++++++---------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 7de42dd..3444d25 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -1,5 +1,4 @@ - -def doKillLeftovers() { +def doDisplayLeftovers() { if (isUnix()) { sh """ # Killing any processes that may interfere with the next build @@ -7,11 +6,23 @@ def doKillLeftovers() { # Don't return an error if the processes don't exist USER=`whoami` ps aux | grep "eclipse" - # pkill -U "${USER}" -9 eclipse || true ps aux | grep "make" - # pkill -U "${USER}" -9 make || true ps aux | grep "arm-linux" - # pkill -U "${USER}" -9 ".*arm-linux.*" || true + """ + } +} + +def doKillLeftovers() { + if (isUnix()) { + doDisplayLeftovers() + sh """ + # Killing any processes that may interfere with the next build + # The build system must be setup to do more than one eclipse build concurrently + # Don't return an error if the processes don't exist + USER=`whoami` + pkill -U "${USER}" -9 eclipse || true + pkill -U "${USER}" -9 make || true + pkill -U "${USER}" -9 ".*arm-linux.*" || true """ } } @@ -106,45 +117,51 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { - doKillLeftovers() - addToolchainPath(toolchainfolder) - doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) - for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo - if (build) { - if (cleanbuild) { - buildtype = "-cleanBuild" + doDisplayLeftovers() + try { + addToolchainPath(toolchainfolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + for (appInfo in apps ) { + (build, cleanbuild, project, variants, exportpathlist) = appInfo + if (build) { + if (cleanbuild) { + buildtype = "-cleanBuild" + } + else { + buildtype = "-build" + } + for (build_variant in variants) { + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } + doExport(exportfolder, exportpathlist, "", build_variant) + } + } } else { - buildtype = "-build" - } - for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } - doExport(exportfolder, exportpathlist, "", build_variant) + // Only export if the project has no variants - signifying that there are files to export which have been prebuilt + if (!variants) { + doExport(exportfolder, exportpathlist, project) } } } - else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt - if (!variants) { - doExport(exportfolder, exportpathlist, project) - } - } + } catch (Exception e) { + error "Build failed - Killing leftovers" + doKillLeftovers() } + doKillLeftovers() } } From 4013a0221269e1f41d38863ebb053448e8ae1f76 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 22:15:22 +0100 Subject: [PATCH 179/558] Display leftovers at the beginning of each build --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 3444d25..c81e200 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,6 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { - doDisplayLeftovers() try { addToolchainPath(toolchainfolder) doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) @@ -132,6 +131,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doDisplayLeftovers() doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { From 488454ae61eb46aefdff2d570a599b982808790b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 22:15:22 +0100 Subject: [PATCH 180/558] Display leftovers at the beginning of each build --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 3444d25..c81e200 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,6 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { - doDisplayLeftovers() try { addToolchainPath(toolchainfolder) doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) @@ -132,6 +131,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + doDisplayLeftovers() doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { From 885d98e8946707b899605e043ede504f4f26fb87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 22:28:42 +0100 Subject: [PATCH 181/558] Timeout builds --- compile/eclipsebuild/eclipseds5build.gsh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index c81e200..cd31301 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -8,6 +8,7 @@ def doDisplayLeftovers() { ps aux | grep "eclipse" ps aux | grep "make" ps aux | grep "arm-linux" + ps aux | grep "sh" """ } } @@ -135,11 +136,14 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + timeout(time: 5, unit: MINUTES) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + sleep(time: 4, unit: SECONDS) } else { bat """ From efdae69ce827bb37e602759eee6aa03b35597a31 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 22:28:42 +0100 Subject: [PATCH 182/558] Timeout builds --- compile/eclipsebuild/eclipseds5build.gsh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index c81e200..cd31301 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -8,6 +8,7 @@ def doDisplayLeftovers() { ps aux | grep "eclipse" ps aux | grep "make" ps aux | grep "arm-linux" + ps aux | grep "sh" """ } } @@ -135,11 +136,14 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + timeout(time: 5, unit: MINUTES) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + sleep(time: 4, unit: SECONDS) } else { bat """ From 3d5af70b41278ae9e07d8f3b99624229aa2221fb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 22:40:33 +0100 Subject: [PATCH 183/558] Timeout builds --- compile/eclipsebuild/eclipseds5build.gsh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index cd31301..5e1c0de 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -51,6 +51,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { if (isUnix()) { result = sh( script: """ cp -r -v "${srcFileExpanded}" "${destFolder}" + if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true ) @@ -61,6 +62,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { else { result = bat( script: ''' copy \"%srcFileExpanded%\" \"%destFolder%\" + if errorlevel 1 EXIT /b 1 ''', returnStatus: true ) @@ -136,14 +138,14 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { - timeout(time: 5, unit: MINUTES) { + timeout(time: 5, unit: 'MINUTES') { sh """ echo Start time: `date` ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` """ } - sleep(time: 4, unit: SECONDS) + sleep(time: 4, unit: 'SECONDS') } else { bat """ From 3f92a1a0b65cd9cac8a97c61ad39d61180c858f5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 17 Sep 2018 22:40:33 +0100 Subject: [PATCH 184/558] Timeout builds --- compile/eclipsebuild/eclipseds5build.gsh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index cd31301..5e1c0de 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -51,6 +51,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { if (isUnix()) { result = sh( script: """ cp -r -v "${srcFileExpanded}" "${destFolder}" + if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true ) @@ -61,6 +62,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { else { result = bat( script: ''' copy \"%srcFileExpanded%\" \"%destFolder%\" + if errorlevel 1 EXIT /b 1 ''', returnStatus: true ) @@ -136,14 +138,14 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" if (isUnix()) { - timeout(time: 5, unit: MINUTES) { + timeout(time: 5, unit: 'MINUTES') { sh """ echo Start time: `date` ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' echo End time: `date` """ } - sleep(time: 4, unit: SECONDS) + sleep(time: 4, unit: 'SECONDS') } else { bat """ From bff5c0a833fb03181736610da871a3498a6e80a9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Sep 2018 00:25:06 +0100 Subject: [PATCH 185/558] Jobs are being killed unexpectedly so remove potential source; Add more timeouts; Extend timeouts --- compile/eclipsebuild/eclipseds5build.gsh | 48 +++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5e1c0de..9fd5293 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -8,7 +8,7 @@ def doDisplayLeftovers() { ps aux | grep "eclipse" ps aux | grep "make" ps aux | grep "arm-linux" - ps aux | grep "sh" + # ps aux | grep "sh" """ } } @@ -121,8 +121,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { try { - addToolchainPath(toolchainfolder) - doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + timeout(time: 2, unit: 'MINUTES') { + addToolchainPath(toolchainfolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + } for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { @@ -134,25 +136,30 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - if (isUnix()) { - timeout(time: 5, unit: 'MINUTES') { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + try { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } + doExport(exportfolder, exportpathlist, "", build_variant) + } catch (Exception e) { + echo "Build failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') + error "Build failed..." } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } - doExport(exportfolder, exportpathlist, "", build_variant) } } } @@ -164,10 +171,9 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } } catch (Exception e) { + // doKillLeftovers() error "Build failed - Killing leftovers" - doKillLeftovers() } - doKillLeftovers() } } From 9c52e610af9667d69015cf5adcd69fef9a426807 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 18 Sep 2018 00:25:06 +0100 Subject: [PATCH 186/558] Jobs are being killed unexpectedly so remove potential source; Add more timeouts; Extend timeouts --- compile/eclipsebuild/eclipseds5build.gsh | 48 +++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 5e1c0de..9fd5293 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -8,7 +8,7 @@ def doDisplayLeftovers() { ps aux | grep "eclipse" ps aux | grep "make" ps aux | grep "arm-linux" - ps aux | grep "sh" + # ps aux | grep "sh" """ } } @@ -121,8 +121,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { try { - addToolchainPath(toolchainfolder) - doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + timeout(time: 2, unit: 'MINUTES') { + addToolchainPath(toolchainfolder) + doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) + } for (appInfo in apps ) { (build, cleanbuild, project, variants, exportpathlist) = appInfo if (build) { @@ -134,25 +136,30 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - if (isUnix()) { - timeout(time: 5, unit: 'MINUTES') { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + try { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } + doExport(exportfolder, exportpathlist, "", build_variant) + } catch (Exception e) { + echo "Build failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') + error "Build failed..." } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } - doExport(exportfolder, exportpathlist, "", build_variant) } } } @@ -164,10 +171,9 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } } catch (Exception e) { + // doKillLeftovers() error "Build failed - Killing leftovers" - doKillLeftovers() } - doKillLeftovers() } } From fadf5e954325f20d50142038be5aa428ad5d31af Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 11 Oct 2018 20:48:16 +0100 Subject: [PATCH 187/558] Fail build if a copy fails --- compile/eclipsebuild/eclipseds5build.gsh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9fd5293..3061fea 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -154,7 +154,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } - doExport(exportfolder, exportpathlist, "", build_variant) + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." + } } catch (Exception e) { echo "Build failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') @@ -166,7 +169,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - doExport(exportfolder, exportpathlist, project) + result = doExport(exportfolder, exportpathlist, project) + if (result != 0) { + error "Failed to copy...." + } } } } From df638458def8228976552ac48bc44e6770b57335 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 11 Oct 2018 20:48:16 +0100 Subject: [PATCH 188/558] Fail build if a copy fails --- compile/eclipsebuild/eclipseds5build.gsh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 9fd5293..3061fea 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -154,7 +154,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } - doExport(exportfolder, exportpathlist, "", build_variant) + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." + } } catch (Exception e) { echo "Build failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') @@ -166,7 +169,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - doExport(exportfolder, exportpathlist, project) + result = doExport(exportfolder, exportpathlist, project) + if (result != 0) { + error "Failed to copy...." + } } } } From 2dfbe7b4c33a2313ab78fc562686d9d3c5dbed1a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 13:32:04 +0100 Subject: [PATCH 189/558] Move meta artifact builder to common scripts subtree area --- artifacts/metaexport.gsh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 artifacts/metaexport.gsh diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.gsh new file mode 100644 index 0000000..368871e --- /dev/null +++ b/artifacts/metaexport.gsh @@ -0,0 +1,24 @@ + +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKSPACE}") { + artifactFolders = "" + for (subJob in subJobs) { + subJobObject = subJob[0] + artifactFolder = subJob[1] + if (subJobObject != null) { + subJobName = subJobObject.getProjectName() + subJobNumber = subJobObject.getNumber() + destFolder = "${exportRootFolder}/${artifactFolder}" + + dir("${destFolder}") { + deleteDir() + } + + echo "Copying artifacts from: Job: ${subJobName} Number: ${subJobNumber} to folder: ${destFolder}" + copyArtifacts fingerprintArtifacts: true, projectName: "${subJobName}", selector: specific("${subJobNumber}"), target: "${destFolder}" + artifactFolders += "${destFolder}/**," + } + } +} + +return this; From 36e3212d79afc20d29b2eed2266e2f0216cfaddf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 24 Oct 2018 14:24:17 +0100 Subject: [PATCH 190/558] Move meta artifact builder to common scripts subtree area --- artifacts/metaexport.gsh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.gsh index 368871e..66a8153 100644 --- a/artifacts/metaexport.gsh +++ b/artifacts/metaexport.gsh @@ -1,6 +1,6 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKSPACE}") { +def combineSubjobArtifacts(subJobs) { artifactFolders = "" for (subJob in subJobs) { subJobObject = subJob[0] @@ -8,7 +8,7 @@ def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKS if (subJobObject != null) { subJobName = subJobObject.getProjectName() subJobNumber = subJobObject.getNumber() - destFolder = "${exportRootFolder}/${artifactFolder}" + destFolder = "${artifactFolder}" dir("${destFolder}") { deleteDir() @@ -19,6 +19,7 @@ def combineSubjobArtifacts(subJobs, artifactFolders, exportRootFolder = "${WORKS artifactFolders += "${destFolder}/**," } } + return artifactFolders; } return this; From 34d5e771d77e356073a4e98857073b508b8ccaee Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 16:24:16 +0000 Subject: [PATCH 191/558] When working out what version prefix to use from a revision - if there are no dots (but there are numbers) default to 1.25000.0. --- versionnumbering/versioninfo.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 1a6a72b..ae2b0ee 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -76,6 +76,7 @@ def getVersionPrefix(sourcebranch): defaultversionprefix = "1.30000.0." mastertypeversionprefix = "1.20000.0." + thirdpartyversionprefix = "1.25000.0." splits = sourcebranch.split('/') if len(splits) < 1: print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) @@ -88,9 +89,16 @@ def getVersionPrefix(sourcebranch): splits2 = re.split(r'(\d+)', sourcebranch) splits = [] + hasdot = False for split in splits2: if split.isdigit(): splits.append(split) + if split == ".": + hasdot = True + + if not hasdot: + print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) + return thirdpartyversionprefix versionprefix = "" if len(splits) < 1: From 40d3111dabc57771442bc02d72a11299231b405f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 16:52:36 +0000 Subject: [PATCH 192/558] Prevent revision number going above 16 bits worth of integer --- versionnumbering/versioninfo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index ae2b0ee..7122dba 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -25,6 +25,7 @@ __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") slaveMachine = os.getenv("NODE_NAME", default="-") +MAXVERSIONNUMBER = 65535 def resetFolder(versionfolder): print("Removing folders and contents of: {0}".format(versionfolder)) @@ -48,7 +49,7 @@ def getGitRevision(gitrootpath, basenumber): print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) - adjustedchangecount = int(changecount) + int(basenumber) + adjustedchangecount = (int(changecount) + int(basenumber) % (MAXVERSIONNUMBER + 1)) print("Adjusted change count: {0}".format(adjustedchangecount)) print("Determining current revision info using git command: {0}".format(gitExecutable)) From 73f0cb98163fbd7a82e4a35cf47b89bedca88db4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 29 Oct 2018 17:05:52 +0000 Subject: [PATCH 193/558] Prevent revision number going above 16 bits worth of integer --- versionnumbering/versioninfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 7122dba..ceae653 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -49,7 +49,7 @@ def getGitRevision(gitrootpath, basenumber): print("Command {0} returned: {1}".format(completed.args, completed.returncode)) changecount = completed.stdout.decode("utf-8").strip() print("Change count: {0}".format(changecount)) - adjustedchangecount = (int(changecount) + int(basenumber) % (MAXVERSIONNUMBER + 1)) + adjustedchangecount = int((int(changecount) + int(basenumber)) % (MAXVERSIONNUMBER + 1)) print("Adjusted change count: {0}".format(adjustedchangecount)) print("Determining current revision info using git command: {0}".format(gitExecutable)) From 7c2c9b22a1a0c43c1c67dcefc683ddb113fd0312 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 12:25:04 +0000 Subject: [PATCH 194/558] Output more git information in the meta version output --- versionnumbering/metaversion.gsh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 9e09b34..634da16 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,9 +33,13 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" + remoteurlFile = "${versionFolder}/remoteurl.txt" + remoteurlFileContents = readFile "${remoteurlFile}" + gitRevisionFile = "${versionFolder}/gitrevision.txt" + gitRevisionFileContents = readFile "${gitRevisionFile}" metabuildStampFile = "${OF}/buildstamps.txt" metabuildStampFileContents = readFile "${metabuildStampFile}" - writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ": " + buildStampFileContents + writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() From 97e0d040efcf6802194126cb9b6623a3236bf186 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 31 Oct 2018 16:28:37 +0000 Subject: [PATCH 195/558] Handle the situation where a build does not provide a remoteurl.txt file --- versionnumbering/metaversion.gsh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.gsh index 634da16..911e094 100644 --- a/versionnumbering/metaversion.gsh +++ b/versionnumbering/metaversion.gsh @@ -33,10 +33,18 @@ def combineSubjobVersions(versionLabelPrefix, versionFolderList, outputFolder = buildStampFile = "${versionFolder}/buildstamp.txt" buildStampFileContents = readFile "${buildStampFile}" + remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = readFile "${remoteurlFile}" + if (fileExists("${remoteurlFile}")) { + remoteurlFileContents = readFile "${remoteurlFile}" + } + else { + remoteurlFileContents = 'Unknown' + } + gitRevisionFile = "${versionFolder}/gitrevision.txt" gitRevisionFileContents = readFile "${gitRevisionFile}" + metabuildStampFile = "${OF}/buildstamps.txt" metabuildStampFileContents = readFile "${metabuildStampFile}" writeFile file: "${metabuildStampFile}", text: metabuildStampFileContents + "\r\n" + "${versionFolder}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents From f3f97107134ead2f6705c8f2916de36e63725e6d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 7 Nov 2018 22:36:03 +0000 Subject: [PATCH 196/558] Only build selected variants --- compile/eclipsebuild/eclipseds5build.gsh | 48 +++++++++++++++--------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 3061fea..078845e 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,14 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild. exportfolder = "${WORKSPACE}/bin") { + toBuild = [] + if (variantstobuild) { + toBuildUntrimmed = variantstobuild.split(',') + toBuildUntrimmed.each { + toBuild.add(it.trim()) + } + } // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { try { @@ -137,26 +144,31 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) try { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } } catch (Exception e) { echo "Build failed - Waiting for a while" From 760bf239a45437585f3b45a7558b617363c941c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 7 Nov 2018 22:39:15 +0000 Subject: [PATCH 197/558] Only build selected variants --- compile/eclipsebuild/eclipseds5build.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.gsh index 078845e..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.gsh +++ b/compile/eclipsebuild/eclipseds5build.gsh @@ -117,7 +117,7 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild. exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}/bin") { toBuild = [] if (variantstobuild) { toBuildUntrimmed = variantstobuild.split(',') From 3d1326836b8da27869d75498f8343ca80dc9cf4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:04:34 +0000 Subject: [PATCH 198/558] Condense autobuild startup boilerplate --- autobuilder.gsh | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 autobuilder.gsh diff --git a/autobuilder.gsh b/autobuilder.gsh new file mode 100644 index 0000000..06a5123 --- /dev/null +++ b/autobuilder.gsh @@ -0,0 +1,39 @@ +// Assume that WORKSPACE environment variable is set + +def autobuildroot = "${WORKSPACE}/autobuild" +def commonscriptsroot = "${autobuildroot}/Common_scripts" + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + cleanWs() + unstash name: 'Clean workspace stash' + } + else { + echo "Not cleaning workspace" + } +} + +// Start up boilderplate +def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autobuildroot}/Common_scripts") { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") + configurationhelper.registerPipelineScriptConfiguration() + +def commonscripts() + return commonscriptsroot +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.gsh") +} + +return this; \ No newline at end of file From 0d839a9d5c03486fbf60ba7cff890a74bad63aac Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:08:35 +0000 Subject: [PATCH 199/558] Condense autobuild startup boilerplate --- autobuilder.gsh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 06a5123..7a7efa1 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -27,8 +27,9 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") configurationhelper.registerPipelineScriptConfiguration() +} -def commonscripts() +def commonscripts() { return commonscriptsroot } From 11b541691331f40f9b6adba02d0f809d525e98c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:13:21 +0000 Subject: [PATCH 200/558] Condense autobuild startup boilerplate --- autobuilder.gsh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 7a7efa1..1657815 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,7 +1,7 @@ // Assume that WORKSPACE environment variable is set -def autobuildroot = "${WORKSPACE}/autobuild" -def commonscriptsroot = "${autobuildroot}/Common_scripts" +String autobuildroot = "${WORKSPACE}/autobuild" +String commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From f33b4931606117107a07ffe2130e93e447a6a7ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:23:19 +0000 Subject: [PATCH 201/558] Condense autobuild startup boilerplate --- autobuilder.gsh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 1657815..88b68a0 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,7 +1,10 @@ // Assume that WORKSPACE environment variable is set -String autobuildroot = "${WORKSPACE}/autobuild" -String commonscriptsroot = "${autobuildroot}/Common_scripts" +@Field +def autobuildroot = "${WORKSPACE}/autobuild" + +@Field +def commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From 80d5325cdb6a5e8940d42ba9d091fc4b16ba15af Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:25:18 +0000 Subject: [PATCH 202/558] Condense autobuild startup boilerplate --- autobuilder.gsh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index 88b68a0..bf42e4c 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -1,10 +1,8 @@ // Assume that WORKSPACE environment variable is set -@Field -def autobuildroot = "${WORKSPACE}/autobuild" +autobuildroot = "${WORKSPACE}/autobuild" -@Field -def commonscriptsroot = "${autobuildroot}/Common_scripts" +commonscriptsroot = "${autobuildroot}/Common_scripts" def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { From cdf5140b4fd74d03bdce6bad0a3d01dd7ceae35b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:31:25 +0000 Subject: [PATCH 203/558] Condense autobuild startup boilerplate --- configuration/configuration.gsh | 2 +- versioncontrol/git.gsh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index 5688482..dbaf921 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -1,4 +1,4 @@ -gh = load("${COMMON_SCRIPTS}/versioncontrol/git.gsh") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.gsh") configurationCollection = [:] diff --git a/versioncontrol/git.gsh b/versioncontrol/git.gsh index 51dbd5d..1462a8e 100644 --- a/versioncontrol/git.gsh +++ b/versioncontrol/git.gsh @@ -1,4 +1,4 @@ -clh = load("${COMMON_SCRIPTS}/commandline/commandline.gsh") +clh = load("${autobuilder.commonscripts()}/commandline/commandline.gsh") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 6116c229d6de51ee7a629ccfb5f30d9a30508dc1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:37:04 +0000 Subject: [PATCH 204/558] Condense autobuild startup boilerplate --- autobuilder.gsh | 4 ++++ configuration/configuration.gsh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/autobuilder.gsh b/autobuilder.gsh index bf42e4c..6ec0e38 100644 --- a/autobuilder.gsh +++ b/autobuilder.gsh @@ -34,6 +34,10 @@ def commonscripts() { return commonscriptsroot } +def getautobuildpath() { + return autobuildroot +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.gsh") } diff --git a/configuration/configuration.gsh b/configuration/configuration.gsh index dbaf921..f131c98 100644 --- a/configuration/configuration.gsh +++ b/configuration/configuration.gsh @@ -8,7 +8,7 @@ def registerConfigurationInfo(String configurationName, String configurationURL, } def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${AUTOBUILD_SCRIPTS}")) + registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } def saveConfigurationInVersionInfo() { From 0e611effff6472605ad838c8fc34ef7acdaf243e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:46:24 +0000 Subject: [PATCH 205/558] Change .gsh suffix to .groovy --- artifacts/{metaexport.gsh => metaexport.groovy} | 0 autobuilder.gsh => autobuilder.groovy | 0 commandline/{commandline.gsh => commandline.groovy} | 0 .../{disableparallelbuilds.gsh => disableparallelbuilds.groovy} | 0 .../eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} | 0 configuration/{configuration.gsh => configuration.groovy} | 0 differences/{json.gsh => json.groovy} | 0 install/{installhelper.gsh => installhelper.groovy} | 0 versioncontrol/{git.gsh => git.groovy} | 0 versionnumbering/{metaversion.gsh => metaversion.groovy} | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename artifacts/{metaexport.gsh => metaexport.groovy} (100%) rename autobuilder.gsh => autobuilder.groovy (100%) rename commandline/{commandline.gsh => commandline.groovy} (100%) rename compile/eclipsebuild/{disableparallelbuilds.gsh => disableparallelbuilds.groovy} (100%) rename compile/eclipsebuild/{eclipseds5build.gsh => eclipseds5build.groovy} (100%) rename configuration/{configuration.gsh => configuration.groovy} (100%) rename differences/{json.gsh => json.groovy} (100%) rename install/{installhelper.gsh => installhelper.groovy} (100%) rename versioncontrol/{git.gsh => git.groovy} (100%) rename versionnumbering/{metaversion.gsh => metaversion.groovy} (100%) diff --git a/artifacts/metaexport.gsh b/artifacts/metaexport.groovy similarity index 100% rename from artifacts/metaexport.gsh rename to artifacts/metaexport.groovy diff --git a/autobuilder.gsh b/autobuilder.groovy similarity index 100% rename from autobuilder.gsh rename to autobuilder.groovy diff --git a/commandline/commandline.gsh b/commandline/commandline.groovy similarity index 100% rename from commandline/commandline.gsh rename to commandline/commandline.groovy diff --git a/compile/eclipsebuild/disableparallelbuilds.gsh b/compile/eclipsebuild/disableparallelbuilds.groovy similarity index 100% rename from compile/eclipsebuild/disableparallelbuilds.gsh rename to compile/eclipsebuild/disableparallelbuilds.groovy diff --git a/compile/eclipsebuild/eclipseds5build.gsh b/compile/eclipsebuild/eclipseds5build.groovy similarity index 100% rename from compile/eclipsebuild/eclipseds5build.gsh rename to compile/eclipsebuild/eclipseds5build.groovy diff --git a/configuration/configuration.gsh b/configuration/configuration.groovy similarity index 100% rename from configuration/configuration.gsh rename to configuration/configuration.groovy diff --git a/differences/json.gsh b/differences/json.groovy similarity index 100% rename from differences/json.gsh rename to differences/json.groovy diff --git a/install/installhelper.gsh b/install/installhelper.groovy similarity index 100% rename from install/installhelper.gsh rename to install/installhelper.groovy diff --git a/versioncontrol/git.gsh b/versioncontrol/git.groovy similarity index 100% rename from versioncontrol/git.gsh rename to versioncontrol/git.groovy diff --git a/versionnumbering/metaversion.gsh b/versionnumbering/metaversion.groovy similarity index 100% rename from versionnumbering/metaversion.gsh rename to versionnumbering/metaversion.groovy From d3ef7cc963fdd7e37c339c8b3bdc255214058009 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 21:54:02 +0000 Subject: [PATCH 206/558] Change .gsh suffix to .groovy --- autobuilder.groovy | 4 ++-- configuration/configuration.groovy | 2 +- versioncontrol/git.groovy | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 6ec0e38..54cfb12 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autob conditionalclean() - configurationhelper = load("${commonscriptsroot}/configuration/configuration.gsh") + configurationhelper = load("${commonscriptsroot}/configuration/configuration.groovy") configurationhelper.registerPipelineScriptConfiguration() } @@ -39,7 +39,7 @@ def getautobuildpath() { } def configurationhelper() { - return load("${commonscripts()}/configuration/configuration.gsh") + return load("${commonscripts()}/configuration/configuration.groovy") } return this; \ No newline at end of file diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index f131c98..1f0c095 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,4 +1,4 @@ -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.gsh") +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") configurationCollection = [:] diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 1462a8e..031094b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,4 +1,4 @@ -clh = load("${autobuilder.commonscripts()}/commandline/commandline.gsh") +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { From 58e7f0c58d9369a218ef4198e0f29179050cd915 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 22:35:36 +0000 Subject: [PATCH 207/558] Remove clean boilerplate --- clean/clean.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 clean/clean.groovy diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file From 76279fba9ef0602d15962df9f8810159c76ed4a6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 23:29:30 +0000 Subject: [PATCH 208/558] Remove versionnumbering boilerplate --- versionnumbering/generate.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 versionnumbering/generate.groovy diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..6e87392 --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,20 @@ + +def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${WORKSPACE}/version", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { + cs = autobuilder.commonscripts() + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${cs}%\\versionnumbering\\versioninfo.py" -s "%${sourcerevision}%" -v "${versionfolder}" -g "${sourcepath}" -b %${basebuildnumber}% -j "%${jobname}%" -n "%${buildid}%" -r "%${releasejobname}%" + """ + } + env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() + env.BUILD_STAMP_COMMA_SEP = readFile("${WORKSPACE}/version/buildstampcommasep.txt").trim() + env.SRC_REVISION = readFile("${WORKSPACE}/version/revision.txt").trim() + configurationhelper.saveConfigurationInVersionInfo(); +} + +return this; \ No newline at end of file From cde5dbabec347a8216e5e83b4288390f39bc94ea Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 10 Nov 2018 23:49:04 +0000 Subject: [PATCH 209/558] Remove differences boilerplate --- differences/do.groovy | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 differences/do.groovy diff --git a/differences/do.groovy b/differences/do.groovy new file mode 100644 index 0000000..9dc1807 --- /dev/null +++ b/differences/do.groovy @@ -0,0 +1,17 @@ + +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { + cs = autobuilder.commonscripts() + + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${cs}%\\differences\\show_release_differences.py" -g "%${sourcepath}%" -s "%${sourcerevision}%" -p "%${previoussourcerevision}%" -j ${jirakey} -o "opposite notcommon" -v "%${versionfolder}%" + """ + } +} + +return this; From db8e1ffe8d148b7b5b17fb9bc7492b4e49b4dc09 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 11 Nov 2018 11:43:28 +0000 Subject: [PATCH 210/558] Remove boilerplate --- versionnumbering/do.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 versionnumbering/do.groovy diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..9b4a42e --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,20 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + filetoconvert = convertdetail[0] + pattern = convertdetail[1] + replacement = convertdetail[2] + if (isUnix()) { + sh """ + "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + """ + } + } +} + +return this; From d73622366001b4c5403058beaa17e8263334b93e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 20:25:15 +0000 Subject: [PATCH 211/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 31 ++++++++++++++++ file/utils.groovy | 16 ++++++++ publishing/utils.groovy | 25 +++++++++++++ versionnumbering/generate.groovy | 11 ++++-- versionnumbering/metaversion.groovy | 57 +++++++++++++++-------------- 5 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 checksum/utils.groovy create mode 100644 file/utils.groovy create mode 100644 publishing/utils.groovy diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..b1a714f --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,31 @@ + +env.CHECKSUMARTIFACTS = "checksums/**" +def CHECKSUMFOLDER = "${WORKSPACE}/checksums" + +// SHA256SUM environment variable must be set to sha256sum utility +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(checksummable) { + if !fileExists("${SHA256SUM}") { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + } + sh """ + rm -rf "${CHECKSUMFOLDER}" + mkdir -p "${CHECKSUMFOLDER}" + """ + for (filepath in checksummable) { + def files = findFiles(glob: filepath) + for (file in files) { + if isUnix() { + sh """#!/bin/bash + "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + """ + } + else { + bat """ + "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + """ + } + } + } +} + diff --git a/file/utils.groovy b/file/utils.groovy new file mode 100644 index 0000000..0ffa47c --- /dev/null +++ b/file/utils.groovy @@ -0,0 +1,16 @@ + +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") + +// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents +def currentContents(file, newline = "${nl}", defaultContents = "") { + def contents = "" + if (fileExists("${file}")) { + contents = readFile(file: "${file}").trim() + newline + } + else { + contents = defaultContents + } + return contents +} + +return this; diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..a10fd76 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,25 @@ + +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") + +env.PUBLISHABLEARTIFACTS = "publishable/**" +def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" +def PUBLISHABLEFILE = "${WORKSPACE}/publishable/list.json" + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(publishable) { + sh """ + rm -rf "${PUBLISHABLEFOLDER}" + mkdir -p "${PUBLISHABLEFOLDER}" + """ + filelist = [] + for (filepath in publishable) { + def files = findFiles(glob: filepath) + for (file in files) { + if (!file.directory) { + filelist.add(file.path) + } + } + } + def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) + writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) +} \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 2776130..6f9f803 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,5 +1,8 @@ -def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${WORKSPACE}/version", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { +env.VERSIONARTIFACTS = "version/**" +def VERSIONFOLDER = "${WORKSPACE}/version" + +def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { cs = autobuilder.commonscripts() if (isUnix()) { sh """ @@ -11,9 +14,9 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" """ } - env.BUILD_STAMP = readFile("${WORKSPACE}/version/buildstamp.txt").trim() - env.BUILD_STAMP_COMMA_SEP = readFile("${WORKSPACE}/version/buildstampcommasep.txt").trim() - env.SRC_REVISION = readFile("${WORKSPACE}/version/revision.txt").trim() + env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() + env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() + env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() configurationhelper.saveConfigurationInVersionInfo(); } diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index 1f6cff6..ff20a86 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -1,5 +1,6 @@ def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") def currentContents(file, newline = "\n", defaultContents = "") { if (fileExists("${file}")) { @@ -26,18 +27,19 @@ def prefixline(rootpath, filepath) { def mergemeta(rootpath, versionFolder, outputFolder) { echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - differencesFile = "${versionFolder}/differences.txt" - newContents = readFile file: "${differencesFile}" - metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + newContents + def differencesFile = "${versionFolder}/differences.txt" + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + buildStampFileContents + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents + def intrevision = 0 if (fileExists("${versionFolder}/revision.txt")) { - revision = readFile("${versionFolder}/revision.txt").trim() + def revision = readFile("${versionFolder}/revision.txt").trim() intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" } @@ -51,29 +53,29 @@ def mergemeta(rootpath, versionFolder, outputFolder) { def appendcomponentinfo(rootpath, versionFolder, outputFolder) { echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - differencesFile = "${versionFolder}/differences.txt" + def differencesFile = "${versionFolder}/differences.txt" if (fileExists("${differencesFile}")) { - newContents = readFile file: "${differencesFile}" - metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents } - buildStampFile = "${versionFolder}/buildstamps.txt" - buildStampFileContents = readFile("${buildStampFile}").trim() + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = readFile("${buildStampFile}").trim() - remoteurlFile = "${versionFolder}/remoteurl.txt" - remoteurlFileContents = currentContents(remoteurlFile, "", 'Unknown').trim() + def remoteurlFile = "${versionFolder}/remoteurl.txt" + def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - gitRevisionFile = "${versionFolder}/gitrevision.txt" - gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + def gitRevisionFile = "${versionFolder}/gitrevision.txt" + def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - releaseJobNameFile = "${versionFolder}/releasejobname.txt" - releaseJobNameContents = currentContents("${releaseJobNameFile}", "", "-").trim() + def releaseJobNameFile = "${versionFolder}/releasejobname.txt" + def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents + def revision = readFile("${versionFolder}/revision.txt").trim() + def intrevision = revision.toInteger() echo "Revision read in: ${intrevision}" return intrevision @@ -82,15 +84,16 @@ def appendcomponentinfo(rootpath, versionFolder, outputFolder) { // Each version folder should have been created using version creation scripts in this Common_scripts folder // as their are well-known files in each folder def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - totalrevision = 0 + def totalrevision = 0 dir("${outputFolder}") { deleteDir() } for (artifactItem in artifactList) { (rootpath, versionFolder) = artifactItem echo "Inspecting folder: ${versionFolder}" - metaversionsignaturefile = "${versionFolder}/metaversion.txt" + def metaversionsignaturefile = "${versionFolder}/metaversion.txt" // If the sub project is a meta job + def intrevision = 0 if (fileExists("${metaversionsignaturefile}")) { intrevision = mergemeta(rootpath, versionFolder, outputFolder) } From 22d46575b08ff9115080aa373110142b09621dc3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 20:35:46 +0000 Subject: [PATCH 212/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 6 ++---- versionnumbering/generate.groovy | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index b1a714f..a961dc8 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,6 +1,5 @@ env.CHECKSUMARTIFACTS = "checksums/**" -def CHECKSUMFOLDER = "${WORKSPACE}/checksums" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace @@ -8,6 +7,7 @@ def generatelist(checksummable) { if !fileExists("${SHA256SUM}") { error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } + def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ rm -rf "${CHECKSUMFOLDER}" mkdir -p "${CHECKSUMFOLDER}" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index a10fd76..2fdbb73 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,12 +1,9 @@ -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - env.PUBLISHABLEARTIFACTS = "publishable/**" -def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" -def PUBLISHABLEFILE = "${WORKSPACE}/publishable/list.json" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { + def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" sh """ rm -rf "${PUBLISHABLEFOLDER}" mkdir -p "${PUBLISHABLEFOLDER}" @@ -21,5 +18,6 @@ def generatelist(publishable) { } } def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) + def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) } \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 6f9f803..cb5e944 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,6 +1,6 @@ env.VERSIONARTIFACTS = "version/**" -def VERSIONFOLDER = "${WORKSPACE}/version" +VERSIONFOLDER = "${WORKSPACE}/version" def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { cs = autobuilder.commonscripts() From 578441471580b8d9d3f7656974f08997638e25c3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 21:29:07 +0000 Subject: [PATCH 213/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index a961dc8..e4c37ca 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,8 +4,20 @@ env.CHECKSUMARTIFACTS = "checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - if !fileExists("${SHA256SUM}") { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + def sha256sumfound = true + if (!fileExists("${SHA256SUM}")) { + def sha256sumfound = false + if (isUnix()) { + def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() + if (sha256sumpath) { + echo "Found sha256sum executable here: + env.SHA256SUM = sha256sumpath + sha256sumfound = true + } + } + if (!sha256sumfound) { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" + } } def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ From 757b0c6cb923dcb6c982c23309304d08e09bc35e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 21:50:35 +0000 Subject: [PATCH 214/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index e4c37ca..63c96eb 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -10,7 +10,7 @@ def generatelist(checksummable) { if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { - echo "Found sha256sum executable here: + echo "Found sha256sum executable here: ${sha256sumpath}" env.SHA256SUM = sha256sumpath sha256sumfound = true } From 93233b975b20ad4e364ee613b25cd8a9c25a85cb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:20:30 +0000 Subject: [PATCH 215/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 63c96eb..0d76756 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -27,7 +27,7 @@ def generatelist(checksummable) { for (filepath in checksummable) { def files = findFiles(glob: filepath) for (file in files) { - if isUnix() { + if (isUnix()) { sh """#!/bin/bash "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ From 6f482426adb732ff969831e44a5eff067fb95f6c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:24:30 +0000 Subject: [PATCH 216/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 0d76756..e358b94 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -6,7 +6,7 @@ env.CHECKSUMARTIFACTS = "checksums/**" def generatelist(checksummable) { def sha256sumfound = true if (!fileExists("${SHA256SUM}")) { - def sha256sumfound = false + sha256sumfound = false if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { From 62a9479b71eb86f4ccf93cb85c6d1e6905cb6216 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:39:48 +0000 Subject: [PATCH 217/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index e358b94..a46cd43 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -25,16 +25,18 @@ def generatelist(checksummable) { mkdir -p "${CHECKSUMFOLDER}" """ for (filepath in checksummable) { + echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { + echo "Checksumming ${file.path}" if (isUnix()) { sh """#!/bin/bash - "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ } else { bat """ - "${SHA256SUM}" "${file}" >> "${CHECKSUMFOLDER}/sha256sum_list" + "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" """ } } From e4538f4bd4ada4e44661e81abda55211d3c844aa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 22:55:21 +0000 Subject: [PATCH 218/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index a46cd43..8829198 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -43,3 +43,4 @@ def generatelist(checksummable) { } } +return this; From 3122037d60b11c0cc635baaaf576e8acd17eb4dd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 23:09:52 +0000 Subject: [PATCH 219/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 8829198..ef80fdd 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,9 +4,14 @@ env.CHECKSUMARTIFACTS = "checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - def sha256sumfound = true - if (!fileExists("${SHA256SUM}")) { - sha256sumfound = false + def sha256sumfound = false + if (env.SHA256SUM) { + if (fileExists("${SHA256SUM}")) { + sha256sumfound = true + } + } + if (!sha256sumfound) + { if (isUnix()) { def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() if (sha256sumpath) { @@ -15,9 +20,9 @@ def generatelist(checksummable) { sha256sumfound = true } } - if (!sha256sumfound) { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" - } + } + if (!sha256sumfound) { + error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } def CHECKSUMFOLDER = "${WORKSPACE}/checksums" sh """ From 24dfa23ce752129658fa465870d2554014ec2904 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 17 Nov 2018 23:20:01 +0000 Subject: [PATCH 220/558] Create a publishable artifact list; Create a checksum list --- checksum/utils.groovy | 1 - publishing/utils.groovy | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ef80fdd..ccf74db 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -33,7 +33,6 @@ def generatelist(checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { - echo "Checksumming ${file.path}" if (isUnix()) { sh """#!/bin/bash "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 2fdbb73..3fcb1f3 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -20,4 +20,6 @@ def generatelist(publishable) { def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) -} \ No newline at end of file +} + +return this; From 44bc4006328b423b56cbe70f6615776124d5ed9d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 09:48:30 +0000 Subject: [PATCH 221/558] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 3 ++- publishing/utils.groovy | 3 ++- versionnumbering/metaversion.groovy | 10 ---------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ccf74db..88496b8 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,5 +1,6 @@ -env.CHECKSUMARTIFACTS = "checksums/**" +def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 3fcb1f3..42ebced 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,5 +1,6 @@ -env.PUBLISHABLEARTIFACTS = "publishable/**" +def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy index ff20a86..44bfbde 100644 --- a/versionnumbering/metaversion.groovy +++ b/versionnumbering/metaversion.groovy @@ -2,16 +2,6 @@ def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") -def currentContents(file, newline = "\n", defaultContents = "") { - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - def prefixline(rootpath, filepath) { def fileContents = readFile("${filepath}").trim() def fileContentsNormalized = fileContents.normalize() From 18d24d64ab7eb9b99ee29515336232b9bc5d42ef Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:24:08 +0000 Subject: [PATCH 222/558] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 meta/folder.groovy diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..d7c91ef --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,10 @@ + +METAFOLDERNAME = "meta" + +def reset() { + dir("${WORKSPACE}/${METAFOLDERNAME}") { + deleteDir() + } +} + +return this; \ No newline at end of file From dea2372d54f908dbfc567f0082f40e695d031436 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:33:10 +0000 Subject: [PATCH 223/558] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index d7c91ef..2f4aaa3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,6 +2,7 @@ METAFOLDERNAME = "meta" def reset() { + echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } From 87d5366db8a7edc18a30812c1d18185e0299c4c0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:43:54 +0000 Subject: [PATCH 224/558] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 88496b8..02a11e0 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -25,7 +25,7 @@ def generatelist(checksummable) { if (!sha256sumfound) { error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" } - def CHECKSUMFOLDER = "${WORKSPACE}/checksums" + def CHECKSUMFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/checksums" sh """ rm -rf "${CHECKSUMFOLDER}" mkdir -p "${CHECKSUMFOLDER}" diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 42ebced..df7697c 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -4,7 +4,7 @@ env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties def generatelist(publishable) { - def PUBLISHABLEFOLDER = "${WORKSPACE}/publishable" + def PUBLISHABLEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/publishable" sh """ rm -rf "${PUBLISHABLEFOLDER}" mkdir -p "${PUBLISHABLEFOLDER}" From d68d375fd11b7620854757de4e59fa65fadf89a2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:46:49 +0000 Subject: [PATCH 225/558] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 2 +- publishing/utils.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 02a11e0..5ec5ba9 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,5 +1,5 @@ -def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // SHA256SUM environment variable must be set to sha256sum utility diff --git a/publishing/utils.groovy b/publishing/utils.groovy index df7697c..72c6566 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,5 +1,5 @@ -def metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" // Generate a list of artifact files that are publishable to outside parties From db14e8e7669021e87d477d1afd94357ae453cf90 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 13:57:47 +0000 Subject: [PATCH 226/558] Create a meta folder seperate from other artifacts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2f4aaa3..ac82af4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,7 +1,7 @@ METAFOLDERNAME = "meta" -def reset() { +def metareset() { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() From 9e6e594dfeb185b438bce8ff006bb3082acfd5d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:09:47 +0000 Subject: [PATCH 227/558] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fd15492..89e2444 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -8,7 +8,7 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace - stash allowEmpty: true, includes: 'autobuild/**/*', name: 'Clean workspace stash', useDefaultExcludes: false + stash allowEmpty: true, includes: 'autobuild/**', name: 'Clean workspace stash', useDefaultExcludes: false cleanWs() unstash name: 'Clean workspace stash' } @@ -17,21 +17,6 @@ def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { } } -// Start up boilderplate -def start(autobuildpath = "${WORKSPACE}/autobuild", commonscriptspath = "${autobuildroot}/Common_scripts") { - echo "Starting...." - - autobuildroot = autobuildpath - commonscriptsroot = commonscriptspath - - conditionalclean() - - lineendings = load("${commonscriptsroot}/platform/lineendings.groovy") - - def ch = load("${commonscriptsroot}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration() -} - def commonscripts() { return commonscriptsroot } @@ -40,6 +25,24 @@ def getautobuildpath() { return autobuildroot } +// Start up boilderplate +def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + def metafolder = load("${commonscripts()}/meta/folder.groovy") + metafolder.metareset() + + def lineendings = load("${commonscripts()}/platform/lineendings.groovy") + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration() +} + def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } From 88d92bea7257ca5b0b47ffdff8abbe7b2144f2ff Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:11:49 +0000 Subject: [PATCH 228/558] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 89e2444..ef5ea7c 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot { +def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath From c318b8448cb6bd98cf5dc79279f8afe1650681c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:25:55 +0000 Subject: [PATCH 229/558] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 2 -- versionnumbering/generate.groovy | 12 ++++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ef5ea7c..bff3738 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -37,8 +37,6 @@ def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset() - def lineendings = load("${commonscripts()}/platform/lineendings.groovy") - def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cb5e944..935f026 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,9 +1,11 @@ -env.VERSIONARTIFACTS = "version/**" -VERSIONFOLDER = "${WORKSPACE}/version" +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") + +env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" +VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { - cs = autobuilder.commonscripts() + def cs = autobuilder.commonscripts() if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" @@ -17,7 +19,9 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - configurationhelper.saveConfigurationInVersionInfo(); + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.saveConfigurationInVersionInfo(); } return this; From 8a3df96e55362f51550673436f67c8db28becba5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:45:45 +0000 Subject: [PATCH 230/558] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 3 +++ differences/do.groovy | 3 +++ 2 files changed, 6 insertions(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index bff3738..090c0f4 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -45,4 +45,7 @@ def configurationhelper() { return load("${commonscripts()}/configuration/configuration.groovy") } +// Everyone wants to know this +autobuilder = this + return this; diff --git a/differences/do.groovy b/differences/do.groovy index b3039a9..7a550a9 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,4 +1,7 @@ +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" + def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { cs = autobuilder.commonscripts() From 944325f5279fe9231cbdc86ef3d80bf6816175d1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 14:48:52 +0000 Subject: [PATCH 231/558] Create a meta folder seperate from other artifacts --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 935f026..f0f93be 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -20,7 +20,7 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - def ch = load("${commonscripts()}/configuration/configuration.groovy") + def ch = load("${cs}/configuration/configuration.groovy") ch.saveConfigurationInVersionInfo(); } From b06a9fd8c1bcfd0e1bd125d149ea8ec1fbad1b75 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:16:23 +0000 Subject: [PATCH 232/558] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 41 +++---------------------- differences/do.groovy | 6 ++-- differences/show_release_differences.py | 20 ++++++------ meta/folder.groovy | 1 + 4 files changed, 19 insertions(+), 49 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 5ec5ba9..add3a50 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -2,50 +2,19 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" -// SHA256SUM environment variable must be set to sha256sum utility // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { - def sha256sumfound = false - if (env.SHA256SUM) { - if (fileExists("${SHA256SUM}")) { - sha256sumfound = true - } - } - if (!sha256sumfound) - { - if (isUnix()) { - def sha256sumpath = sh(returnStdout: true, script: 'which sha256sum').trim() - if (sha256sumpath) { - echo "Found sha256sum executable here: ${sha256sumpath}" - env.SHA256SUM = sha256sumpath - sha256sumfound = true - } - } - } - if (!sha256sumfound) { - error "Please define the path to a sha256sum utility in the SHA256SUM environment variable" - } - def CHECKSUMFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/checksums" - sh """ - rm -rf "${CHECKSUMFOLDER}" - mkdir -p "${CHECKSUMFOLDER}" - """ + sums = "" for (filepath in checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { - if (isUnix()) { - sh """#!/bin/bash - "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" - """ - } - else { - bat """ - "${SHA256SUM}" "${file.path}" >> "${CHECKSUMFOLDER}/sha256sum_list" - """ - } + def sum = sha1(file: file.path) + echo "Sum from file ${file.path} is ${sum}" + sums += "${sum}${nl}" } } + writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) } return this; diff --git a/differences/do.groovy b/differences/do.groovy index 7a550a9..8866400 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -2,17 +2,17 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, versionfolder = "${WORKSPACE}/version") { +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEARTIFACTS}") { cs = autobuilder.commonscripts() if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -v "${versionfolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 2d3ee0d..0813356 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,9 +29,9 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(versionfolder): - print("Creating folder (if not already there): {0}".format(versionfolder)) - os.makedirs(versionfolder, exist_ok = True) +def makeFolder(difffolder): + print("Creating folder (if not already there): {0}".format(difffolder)) + os.makedirs(difffolder, exist_ok = True) def printDivider(outfh, times = 1, length = 100): for _ in range(times): @@ -172,7 +172,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="", help="version folder where differences file stored [default: %(default)s]") + parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +183,14 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - versionfolder = args.versionfolder + difffolder = args.difffolder print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", versionfolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) - if versionfolder: - makeFolder(versionfolder) - outfh = open(os.path.join(versionfolder, "differences.txt"), 'w') + if difffolder: + makeFolder(difffolder) + outfh = open(os.path.join(difffolder, "differences.txt"), 'w') else: outfh = sys.stdout @@ -209,7 +209,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if versionfolder: + if difffolder: outfh.close() return 0 except Exception as e: diff --git a/meta/folder.groovy b/meta/folder.groovy index ac82af4..6b0b21c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,5 +1,6 @@ METAFOLDERNAME = "meta" +env.METAARTIFACTS = "${METAFOLDERNAME}/**" def metareset() { echo "Resetting meta folder" From 473edb7ad0953f7510df56dcd69df867b9464dc6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:23:39 +0000 Subject: [PATCH 233/558] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index add3a50..ec3b10a 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -4,14 +4,14 @@ env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { + load("${autobuilder.commonscripts()}/platform/lineendings.groovy") sums = "" for (filepath in checksummable) { echo "Checksumming ${filepath}" def files = findFiles(glob: filepath) for (file in files) { def sum = sha1(file: file.path) - echo "Sum from file ${file.path} is ${sum}" - sums += "${sum}${nl}" + sums += "${sum} ${file.path}${nl}" // Format suitable for sha1sum utility } } writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) From 7ee6d66922e60919bd3708b37082f3e09c0f3a03 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:27:40 +0000 Subject: [PATCH 234/558] Create a meta folder seperate from other artifacts --- checksum/utils.groovy | 1 + versionnumbering/generate.groovy | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index ec3b10a..748472f 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,6 +1,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" +CHECKSUMFOLDER = "${metafolder.METAFOLDERNAME}/checksums" // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(checksummable) { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f0f93be..cdcf1c2 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,6 +1,5 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") - env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" From 818e4995f73c4491ae70a4d554f3d9582d4a3b83 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 15:36:24 +0000 Subject: [PATCH 235/558] Create a meta folder seperate from other artifacts --- differences/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 8866400..9e572af 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,10 +1,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" +DIFFERENCEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/differences" -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEARTIFACTS}") { +def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEFOLDER}") { cs = autobuilder.commonscripts() - if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" From c3458cde4f8152cc1dff854173b2af6a40df9cc1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 18 Nov 2018 21:23:41 +0000 Subject: [PATCH 236/558] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 5 ++++- product/info.groovy | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 product/info.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 090c0f4..fc298ba 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -37,6 +37,9 @@ def start(autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset() + def productinfo = load("${commonscripts()}/product/info.groovy") + productinfo.setname(productname, jirakey) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() } diff --git a/product/info.groovy b/product/info.groovy new file mode 100644 index 0000000..c41fc80 --- /dev/null +++ b/product/info.groovy @@ -0,0 +1,16 @@ + +metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") +PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" +env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" + +def setname(productname = "", jirakey = "") { + def productdata = [ + 'name' : productname.trim(), + 'jirakey' : jirakey.trim() + ] + def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) + def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" + writeJSON(file: PRODUCTFILE, json: outJson, pretty: 4) +} + +return this; \ No newline at end of file From 9b95e26e3cca1498f1e35e48615697df70899b67 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 10:52:04 +0000 Subject: [PATCH 237/558] Create a meta folder seperate from other artifacts --- autobuilder.groovy | 4 ++-- product/info.groovy | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fc298ba..ae3af20 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -38,7 +38,7 @@ def start(productname = "", jirakey = "", autobuildpath = autobuildroot, commons metafolder.metareset() def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(productname, jirakey) + productinfo.setname(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/product/info.groovy b/product/info.groovy index c41fc80..05ed92a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -3,9 +3,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -def setname(productname = "", jirakey = "") { +def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { def productdata = [ 'name' : productname.trim(), + 'subcomponent' : subcomponentname.trim() 'jirakey' : jirakey.trim() ] def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) From e4c616c22bead7d34c1df20435a8348898fbca35 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 10:53:23 +0000 Subject: [PATCH 238/558] Create a meta folder seperate from other artifacts --- product/info.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 05ed92a..a908e7e 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -6,8 +6,8 @@ env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { def productdata = [ 'name' : productname.trim(), - 'subcomponent' : subcomponentname.trim() - 'jirakey' : jirakey.trim() + 'subcomponent' : subcomponentname.trim(), + 'jirakey' : jirakey.trim(), ] def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" From 04a1f3e243fa3f6d2691a76927c11bdb434f869f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:32:14 +0000 Subject: [PATCH 239/558] Create a meta folder seperate from other artifacts --- product/info.groovy | 40 ++++++++++++--- versionnumbering/generate.groovy | 83 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 7 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index a908e7e..71cf6fa 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,17 +1,43 @@ +load("${autobuilder.commonscripts()}/json/interface.groovy") + metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" +class ProductInfo extends JsonSerialised { + private String productname + private String subcomponentname + private String jirakey + + ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + productname = productName + subcomponentname = subcomponentName + jirakey = jiraKey + } + + def serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +theProductInfo = new ProductInfo() + def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - def productdata = [ - 'name' : productname.trim(), - 'subcomponent' : subcomponentname.trim(), - 'jirakey' : jirakey.trim(), - ] - def outJson = readJSON text: groovy.json.JsonOutput.toJson(productdata) + theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) + def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - writeJSON(file: PRODUCTFILE, json: outJson, pretty: 4) + theProductInfo.saveJsonSerialised(PRODUCTFILE) } return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cdcf1c2..ad6214d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -3,6 +3,89 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +class GitInfo { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo +} + +class JenkinsBuildInfo { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate +} + +class BuildStampInfo { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep +} + +class ReleaseInfo { + private String releasejobname +} + +class VersionInfo { + private ComponentInfo componentinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + + private VersionInfo[] associatedversions +} + + def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { + + + version.generate("${SOURCE_REVISION}", "${SOURCE_PATH}", "${BASEBUILDNUMBER}", "${RELEASE_JOB_NAME}") + + VersionInfo() { + } + def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): + with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: + f.write(buildjobname) + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: + f.write(buildnumber) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: + f.write(slaveMachine) + + with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: + f.write(str(datetime.datetime.now())) + + with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: + f.write(releasejobname) + +} + def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From 6d14319d823ee8856103cf83864b28e0d6513b60 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:34:36 +0000 Subject: [PATCH 240/558] Create a meta folder seperate from other artifacts --- json/interface.groovy | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 json/interface.groovy diff --git a/json/interface.groovy b/json/interface.groovy new file mode 100644 index 0000000..8962578 --- /dev/null +++ b/json/interface.groovy @@ -0,0 +1,7 @@ + +abstract class JsonSerialised { + def serialisationdata() + def saveJsonSerialised(String outputfile, int prettylevel = 4) +} + +return this; From cf49dbc254fac6f6fefa021fb5cd6648c27339a9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 14:39:03 +0000 Subject: [PATCH 241/558] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 71cf6fa..979d0b5 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -1,5 +1,5 @@ -load("${autobuilder.commonscripts()}/json/interface.groovy") +def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" From e8bf65688350e5838ab13a81514b5fed8d88bde7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:22:46 +0000 Subject: [PATCH 242/558] Create a meta folder seperate from other artifacts --- json/interface.groovy | 4 ++-- product/info.groovy | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index 8962578..d238707 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,7 +1,7 @@ abstract class JsonSerialised { - def serialisationdata() - def saveJsonSerialised(String outputfile, int prettylevel = 4) + def serialisationdata(); + def saveJsonSerialised(String outputfile, int prettylevel = 4); } return this; diff --git a/product/info.groovy b/product/info.groovy index 979d0b5..9b2551c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -11,6 +11,7 @@ class ProductInfo extends JsonSerialised { private String jirakey ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + super() productname = productName subcomponentname = subcomponentName jirakey = jiraKey From d19b4be96d2d36c8bddebaea9fa119667c68af17 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:25:14 +0000 Subject: [PATCH 243/558] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 9b2551c..783b84c 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo extends JsonSerialised { +class ProductInfo extends ji.JsonSerialised { private String productname private String subcomponentname private String jirakey From 8e1175a952781fb3e2501b4e27702e8948d53880 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:26:48 +0000 Subject: [PATCH 244/558] Create a meta folder seperate from other artifacts --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 783b84c..8b8d3e2 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,7 +5,7 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo extends ji.JsonSerialised { +class ProductInfo { private String productname private String subcomponentname private String jirakey From 729b4f1ef2dbcabb7f2845a8a9dac50a9595208d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:29:20 +0000 Subject: [PATCH 245/558] Create a meta folder seperate from other artifacts --- json/interface.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index d238707..4cd2628 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,7 +1,7 @@ abstract class JsonSerialised { - def serialisationdata(); - def saveJsonSerialised(String outputfile, int prettylevel = 4); + abstract void serialisationdata(); + abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } return this; From 2c45a39755ddfebf978cff02989f19ca1582c5b7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 15:37:04 +0000 Subject: [PATCH 246/558] Create a meta folder seperate from other artifacts --- product/info.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/product/info.groovy b/product/info.groovy index 8b8d3e2..536d336 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -26,9 +26,9 @@ class ProductInfo { return data } - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + def saveJsonSerialised(script, String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) } } @@ -38,7 +38,7 @@ def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, ji theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(PRODUCTFILE) + theProductInfo.saveJsonSerialised(this, PRODUCTFILE) } return this; \ No newline at end of file From 284b6aab835ff591c1ea778587d08a8a5794b792 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 16:03:24 +0000 Subject: [PATCH 247/558] Create a meta folder seperate from other artifacts --- json/interface.groovy | 26 +++++++++++ versionnumbering/generate.groovy | 75 +------------------------------- 2 files changed, 28 insertions(+), 73 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index 4cd2628..fae1fe3 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -4,4 +4,30 @@ abstract class JsonSerialised { abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } +class GitInfo { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo +} + +class JenkinsBuildInfo { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate +} + +class BuildStampInfo { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep +} + +class ReleaseInfo { + private String releasejobname +} + return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ad6214d..3d498e9 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,34 +1,10 @@ +def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") + metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -class GitInfo { - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo -} - -class JenkinsBuildInfo { - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate -} - -class BuildStampInfo { - private String buildstampprefix - private int revision - private String buildstamp - private String buildstampcommasep -} - -class ReleaseInfo { - private String releasejobname -} - class VersionInfo { private ComponentInfo componentinfo private GitInfo gitinfo @@ -37,53 +13,6 @@ class VersionInfo { private ReleaseInfo releaseinfo private VersionInfo[] associatedversions -} - - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { - - - version.generate("${SOURCE_REVISION}", "${SOURCE_PATH}", "${BASEBUILDNUMBER}", "${RELEASE_JOB_NAME}") - - VersionInfo() { - } - def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - - with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: - f.write(releasejobname) - } def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { From 2d709eb4dc9631e3a93625915fd7de73c3643f28 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 16:17:20 +0000 Subject: [PATCH 248/558] Create a meta folder seperate from other artifacts --- json/interface.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/json/interface.groovy b/json/interface.groovy index fae1fe3..c9cd9f9 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -4,7 +4,7 @@ abstract class JsonSerialised { abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } -class GitInfo { +public class GitInfo { private String sourcepath private String sourcerevision private String gitrevision @@ -12,21 +12,21 @@ class GitInfo { private String remoteinfo } -class JenkinsBuildInfo { +public class JenkinsBuildInfo { private String buildjobname private int buildnumber private String buildmachine private String builddate } -class BuildStampInfo { +public class BuildStampInfo { private String buildstampprefix private int revision private String buildstamp private String buildstampcommasep } -class ReleaseInfo { +public class ReleaseInfo { private String releasejobname } From 46cd0d3d870c8807db798a5be3de62e2e1ee8966 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:30:30 +0000 Subject: [PATCH 249/558] Pass script object to custom objects --- autobuilder.groovy | 4 ++-- product/info.groovy | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index ae3af20..ffaaf9f 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -38,7 +38,7 @@ def start(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jira metafolder.metareset() def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(productname, subcomponentname, jirakey) + productinfo.setname(script, productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/product/info.groovy b/product/info.groovy index 536d336..1e0af6d 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -5,12 +5,14 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" -class ProductInfo { +class ProductInfo implements Serializable { + private script private String productname private String subcomponentname private String jirakey - ProductInfo(productName = "", subcomponentName = "", jiraKey = "") { + ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { + this.script = script super() productname = productName subcomponentname = subcomponentName @@ -26,7 +28,7 @@ class ProductInfo { return data } - def saveJsonSerialised(script, String outputfile, int prettylevel = 4) { + def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) } @@ -34,11 +36,11 @@ class ProductInfo { theProductInfo = new ProductInfo() -def setname(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theProductInfo = new ProductInfo(productname, subcomponentname, jirakey) +def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { + theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(this, PRODUCTFILE) + theProductInfo.saveJsonSerialised(PRODUCTFILE) } return this; \ No newline at end of file From 0ee5068c158c3d5f7dbc08c2e4a44921e432e297 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:32:24 +0000 Subject: [PATCH 250/558] Pass script object to custom objects --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 1e0af6d..19f5df0 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -12,8 +12,8 @@ class ProductInfo implements Serializable { private String jirakey ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { - this.script = script super() + this.script = script productname = productName subcomponentname = subcomponentName jirakey = jiraKey From 8b99f633acde702a6efaf28eccb456144fb01d60 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:34:40 +0000 Subject: [PATCH 251/558] Pass script object to custom objects --- product/info.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/product/info.groovy b/product/info.groovy index 19f5df0..78e281a 100644 --- a/product/info.groovy +++ b/product/info.groovy @@ -34,7 +34,7 @@ class ProductInfo implements Serializable { } } -theProductInfo = new ProductInfo() +theProductInfo = null def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) From ea8e58d3d1b4d3e6a0667defe83ed295addc6795 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 17:52:07 +0000 Subject: [PATCH 252/558] Pass script object to custom objects --- versionnumbering/generate.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3d498e9..3d35e23 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -5,8 +5,10 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +def pi = load("${autobuilder.commonscripts()}/product/info.groovy") + class VersionInfo { - private ComponentInfo componentinfo + private ProductInfo productinfo private GitInfo gitinfo private JenkinsBuildInfo jenkinsbuildinfo private BuildStampInfo buildstampinfo From 94ddf17df7b3d488232ab983c66c98a963ef3568 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Nov 2018 18:25:17 +0000 Subject: [PATCH 253/558] Pass script object to custom objects --- json/interface.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json/interface.groovy b/json/interface.groovy index c9cd9f9..58f9eb7 100644 --- a/json/interface.groovy +++ b/json/interface.groovy @@ -1,5 +1,5 @@ -abstract class JsonSerialised { +abstract class GISJsonSerialised { abstract void serialisationdata(); abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); } From bbc3da7e374a9fbf188c3ab7826563214ea94698 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:18:27 +0000 Subject: [PATCH 254/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- autobuilder.groovy | 6 +- json/interface.groovy | 33 ------ meta/folder.groovy | 180 ++++++++++++++++++++++++++++++- product/info.groovy | 46 -------- versionnumbering/generate.groovy | 13 --- 5 files changed, 181 insertions(+), 97 deletions(-) delete mode 100644 json/interface.groovy delete mode 100644 product/info.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index ffaaf9f..dfc470b 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -35,10 +35,8 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() def metafolder = load("${commonscripts()}/meta/folder.groovy") - metafolder.metareset() - - def productinfo = load("${commonscripts()}/product/info.groovy") - productinfo.setname(script, productname, subcomponentname, jirakey) + metafolder.metareset(script) + metafolder.setproductinfo(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/json/interface.groovy b/json/interface.groovy deleted file mode 100644 index 58f9eb7..0000000 --- a/json/interface.groovy +++ /dev/null @@ -1,33 +0,0 @@ - -abstract class GISJsonSerialised { - abstract void serialisationdata(); - abstract void saveJsonSerialised(String outputfile, int prettylevel = 4); -} - -public class GitInfo { - private String sourcepath - private String sourcerevision - private String gitrevision - private String remoteurl - private String remoteinfo -} - -public class JenkinsBuildInfo { - private String buildjobname - private int buildnumber - private String buildmachine - private String builddate -} - -public class BuildStampInfo { - private String buildstampprefix - private int revision - private String buildstamp - private String buildstampcommasep -} - -public class ReleaseInfo { - private String releasejobname -} - -return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 6b0b21c..a8bc619 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,11 +2,189 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" -def metareset() { +class GISJsonSerialised implements Serializable { + protected script + + GISJsonSerialied(script) { + this.script = script + } + + abstract void serialisationdata(); + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + private String productname + private String subcomponentname + private String jirakey + + def set(script, productname, subcomponentname, jirakey) { + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jiraKey + } + + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + set(productname, subcomponentname, jirakey) + } + + def serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + super(script) + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + def serialisationdata() { + def data = [ + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + + JenkinsBuildInfo(script, buildjobname = "", buildnumber = 0, buildmachine = "", builddate = "") { + super(script) + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + def serialisationdata() { + def data = [ + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + private String buildstampprefix + private int revision + private String buildstamp + private String buildstampcommasep + + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + super(script) + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + + def serialisationdata() { + def data = [ + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + 'buildstampcommasep' : buildstampcommasep, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + private String releasejobname + + BuildStampInfo(script, releasejobname = "") { + super(script) + this.releasejobname = releasejobname + } + + def serialisationdata() { + def data = [ + 'releasejobname' : releasejobname, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private ProductInfo productinfo + private GitInfo gitinfo + private JenkinsBuildInfo jenkinsbuildinfo + private BuildStampInfo buildstampinfo + private ReleaseInfo releaseinfo + + private MetaInfo[] associatedmeta + + MetaInfo(script) { + super(script) + productinfo(script) + gitinfo(script) + jenkinsbuildinfo(script) + buildstampinfo(script) + releaseinfo(script) + } + + def serialisationdata() { + def data = [ + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + ] + return datasetProductInfo + } + + def setproductinfo(productname, subcomponentname, jirakey) { + productinfo.set(productname, subcomponentname, jirakey) + } + + +MetaInfo theMetaData = new MetaInfo(this) + +def metareset(script) { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } + theMetaData = new MetaInfo(this) } +def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { + theMetaData.setproductinfo(productname, subcomponentname, jirakey) +} + + + return this; \ No newline at end of file diff --git a/product/info.groovy b/product/info.groovy deleted file mode 100644 index 78e281a..0000000 --- a/product/info.groovy +++ /dev/null @@ -1,46 +0,0 @@ - -def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") - -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -PRODUCTFOLDER = "${metafolder.METAFOLDERNAME}/product" -env.PRODUCTARTIFACTS = "${PRODUCTFOLDER}/**" - -class ProductInfo implements Serializable { - private script - private String productname - private String subcomponentname - private String jirakey - - ProductInfo(script, productName = "", subcomponentName = "", jiraKey = "") { - super() - this.script = script - productname = productName - subcomponentname = subcomponentName - jirakey = jiraKey - } - - def serialisationdata() { - def data = [ - 'productname' : productname, - 'subcomponentname' : subcomponentname, - 'jirakey' : jirakey, - ] - return data - } - - def saveJsonSerialised(String outputfile, int prettylevel = 4) { - def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) - script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) - } -} - -theProductInfo = null - -def setname(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theProductInfo = new ProductInfo(script, productname, subcomponentname, jirakey) - - def PRODUCTFILE = "${PRODUCTFOLDER}/info.json" - theProductInfo.saveJsonSerialised(PRODUCTFILE) -} - -return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3d35e23..48f8bb6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,22 +1,9 @@ -def ji = load("${autobuilder.commonscripts()}/json/interface.groovy") - metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.VERSIONARTIFACTS = "${metafolder.METAFOLDERNAME}/version/**" VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" def pi = load("${autobuilder.commonscripts()}/product/info.groovy") -class VersionInfo { - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo - - private VersionInfo[] associatedversions -} - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From bda26208b76316b402735615e9b2446ee8690226 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:37:27 +0000 Subject: [PATCH 255/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- versionnumbering/generate.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 48f8bb6..b576201 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -2,8 +2,6 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -def pi = load("${autobuilder.commonscripts()}/product/info.groovy") - def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { def cs = autobuilder.commonscripts() if (isUnix()) { From 4b0fa46e8099d9d780e7a6faffae35c74d476ef1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:45:39 +0000 Subject: [PATCH 256/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8bc619..0e800a4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -3,7 +3,7 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" class GISJsonSerialised implements Serializable { - protected script + def script GISJsonSerialied(script) { this.script = script From c0b24175c608579694d653c92d462856191b533c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:47:17 +0000 Subject: [PATCH 257/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0e800a4..ca4680f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -5,7 +5,7 @@ env.METAARTIFACTS = "${METAFOLDERNAME}/**" class GISJsonSerialised implements Serializable { def script - GISJsonSerialied(script) { + GISJsonSerialised(script) { this.script = script } From 923c0c69db923ef83dbfb1a28fe6a061312cdc57 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:49:13 +0000 Subject: [PATCH 258/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ca4680f..65d4708 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -124,7 +124,7 @@ class BuildStampInfo extends GISJsonSerialised { class ReleaseInfo extends GISJsonSerialised { private String releasejobname - BuildStampInfo(script, releasejobname = "") { + ReleaseInfo(script, releasejobname = "") { super(script) this.releasejobname = releasejobname } From fc8c1c130ab8e261c226f96fa3ede556a3520069 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 17:51:29 +0000 Subject: [PATCH 259/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 65d4708..46ae4f3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -169,7 +169,7 @@ class MetaInfo extends GISJsonSerialised { def setproductinfo(productname, subcomponentname, jirakey) { productinfo.set(productname, subcomponentname, jirakey) } - +} MetaInfo theMetaData = new MetaInfo(this) @@ -185,6 +185,4 @@ def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_N theMetaData.setproductinfo(productname, subcomponentname, jirakey) } - - return this; \ No newline at end of file From f792738a9aab80f3a2f8d8504d6646742c56f59d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:03:51 +0000 Subject: [PATCH 260/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 46ae4f3..f13cdaa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -2,14 +2,14 @@ METAFOLDERNAME = "meta" env.METAARTIFACTS = "${METAFOLDERNAME}/**" -class GISJsonSerialised implements Serializable { +abstract class GISJsonSerialised implements Serializable { def script GISJsonSerialised(script) { this.script = script } - abstract void serialisationdata(); + abstract Map serialisationdata(); def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -33,7 +33,7 @@ class ProductInfo extends GISJsonSerialised { set(productname, subcomponentname, jirakey) } - def serialisationdata() { + Map serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, @@ -59,7 +59,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - def serialisationdata() { + Map serialisationdata() { def data = [ 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, @@ -85,7 +85,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - def serialisationdata() { + Map serialisationdata() { def data = [ 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, @@ -110,7 +110,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - def serialisationdata() { + Map serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, 'revision' : revision, @@ -129,7 +129,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - def serialisationdata() { + Map serialisationdata() { def data = [ 'releasejobname' : releasejobname, ] @@ -153,15 +153,25 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo(script) buildstampinfo(script) releaseinfo(script) + associatedmeta = [] } - def serialisationdata() { + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(serialisationdata(metaitem)) + } + return associated + } + + Map serialisationdata() { def data = [ 'productinfo' : productinfo.serialisationdata(), 'gitinfo' : gitinfo.serialisationdata(), 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), + 'associatedmeta' : associatedmetaserialised(), ] return datasetProductInfo } From c7bb9095ca36afbe68a120d6c71f17422f5235ba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:38:55 +0000 Subject: [PATCH 261/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f13cdaa..afcd6f7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -148,11 +148,11 @@ class MetaInfo extends GISJsonSerialised { MetaInfo(script) { super(script) - productinfo(script) - gitinfo(script) - jenkinsbuildinfo(script) - buildstampinfo(script) - releaseinfo(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfoo = new JenkinsBuildInfo(script) + buildstampinfoo = new BuildStampInfo(script) + releaseinfoo = new ReleaseInfo(script) associatedmeta = [] } From e59f3b38bc25bbbe424d298bbcb2410b7e1eb5af Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 18:41:46 +0000 Subject: [PATCH 262/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index afcd6f7..ad3636d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -22,7 +22,7 @@ class ProductInfo extends GISJsonSerialised { private String subcomponentname private String jirakey - def set(script, productname, subcomponentname, jirakey) { + def set(productname, subcomponentname, jirakey) { this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jiraKey From 9f5a473fcf768acec6f62a8d98997a2102da0638 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 19:38:29 +0000 Subject: [PATCH 263/558] Move meta data to one groovy file because of the difficulty of including/importing groovy classes --- autobuilder.groovy | 2 +- meta/folder.groovy | 62 ++++++++++++++++++++++++---------------------- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index dfc470b..fab812a 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -36,7 +36,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def metafolder = load("${commonscripts()}/meta/folder.groovy") metafolder.metareset(script) - metafolder.setproductinfo(productname, subcomponentname, jirakey) + metafolder.theMetaData.productinfo.set(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration() diff --git a/meta/folder.groovy b/meta/folder.groovy index ad3636d..d4caa5d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -9,7 +9,7 @@ abstract class GISJsonSerialised implements Serializable { this.script = script } - abstract Map serialisationdata(); + abstract LinkedHashMap serialisationdata(); def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -22,18 +22,20 @@ class ProductInfo extends GISJsonSerialised { private String subcomponentname private String jirakey + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + def set(productname, subcomponentname, jirakey) { this.productname = productname this.subcomponentname = subcomponentname - this.jirakey = jiraKey + this.jirakey = jirakey } - ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { - super(script) - set(productname, subcomponentname, jirakey) - } - - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, 'subcomponentname' : subcomponentname, @@ -59,7 +61,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, 'sourcerevision' : sourcerevision, @@ -85,7 +87,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, 'buildnumber' : buildnumber, @@ -110,7 +112,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, 'revision' : revision, @@ -129,7 +131,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - Map serialisationdata() { + LinkedHashMap serialisationdata() { def data = [ 'releasejobname' : releasejobname, ] @@ -138,34 +140,40 @@ class ReleaseInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private ProductInfo productinfo - private GitInfo gitinfo - private JenkinsBuildInfo jenkinsbuildinfo - private BuildStampInfo buildstampinfo - private ReleaseInfo releaseinfo + private String format = '0.1' + ProductInfo productinfo + GitInfo gitinfo + JenkinsBuildInfo jenkinsbuildinfo + BuildStampInfo buildstampinfo + ReleaseInfo releaseinfo - private MetaInfo[] associatedmeta + private ArrayList associatedmeta MetaInfo(script) { super(script) productinfo = new ProductInfo(script) gitinfo = new GitInfo(script) - jenkinsbuildinfoo = new JenkinsBuildInfo(script) - buildstampinfoo = new BuildStampInfo(script) - releaseinfoo = new ReleaseInfo(script) - associatedmeta = [] + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + associatedmeta = new ArrayList() } List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { - associated.add(serialisationdata(metaitem)) + associated.add(metaitem.serialisationdata()) } return associated } - Map serialisationdata() { + void addAssociatedMetaInfo(mi) { + associatedmeta.add(mi) + } + + LinkedHashMap serialisationdata() { def data = [ + 'format' : format, 'productinfo' : productinfo.serialisationdata(), 'gitinfo' : gitinfo.serialisationdata(), 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), @@ -173,7 +181,7 @@ class MetaInfo extends GISJsonSerialised { 'releaseinfo' : releaseinfo.serialisationdata(), 'associatedmeta' : associatedmetaserialised(), ] - return datasetProductInfo + return data } def setproductinfo(productname, subcomponentname, jirakey) { @@ -191,8 +199,4 @@ def metareset(script) { theMetaData = new MetaInfo(this) } -def setproductinfo(productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY) { - theMetaData.setproductinfo(productname, subcomponentname, jirakey) -} - return this; \ No newline at end of file From e3942ba1fee975bdbe6989192d258ddf2fa17664 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 20:25:33 +0000 Subject: [PATCH 264/558] Start using meta info for storing configuration --- autobuilder.groovy | 13 ++++---- configuration/configuration.groovy | 33 ++++---------------- meta/folder.groovy | 48 ++++++++++++++++++++++++++---- versionnumbering/generate.groovy | 3 -- 4 files changed, 57 insertions(+), 40 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index fab812a..a82e891 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolder = load("${commonscripts()}/meta/folder.groovy") + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -26,7 +28,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -34,12 +36,13 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - metafolder.metareset(script) - metafolder.theMetaData.productinfo.set(productname, subcomponentname, jirakey) + def meta = metafolder.metareset(script) + meta.productinfo.set(productname, subcomponentname, jirakey) def ch = load("${commonscripts()}/configuration/configuration.groovy") - ch.registerPipelineScriptConfiguration() + ch.registerPipelineScriptConfiguration(meta) + + return meta } def configurationhelper() { diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index 1f0c095..30b7086 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -1,37 +1,16 @@ gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") -configurationCollection = [:] - -def registerConfigurationInfo(String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { - def ConfigurationInfo = [configurationURL, configurationVersion, configurationSHA] - configurationCollection.put(configurationName, ConfigurationInfo) +def registerConfigurationInfo(meta, String configurationName, String configurationURL, String configurationVersion, String configurationSHA) { + meta.addConfigurationInfo(configurationName, configurationURL, configurationVersion, configurationSHA) } -def registerPipelineScriptConfiguration() { - registerConfigurationInfo('pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) +def registerPipelineScriptConfiguration(meta) { + registerConfigurationInfo(meta, 'pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) } -def saveConfigurationInVersionInfo() { - dir("${WORKSPACE}/version") { - echo "Writing configuration info to ${WORKSPACE}/version/configuration.txt" - writeFile file: 'configuration.txt', text: getFormattedConfiguration() - } -} - -String getFormattedConfiguration() { - String formattedInfo = "" - def keys = configurationCollection.keySet().sort() - for (key in keys) { - (configurationURL, configurationVersion, configurationSHA) = configurationCollection.get(key) - formattedLine = String.format("%s:%s,%s,%s%n", key, configurationURL, configurationVersion, configurationSHA) - formattedInfo += formattedLine - } - return formattedInfo -} - -def gitCheckoutAndRegister(String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { gh.gitCheckout(gitURL, revision, targetDir, repoRoot, cloneDepth, shallowClone) - registerConfigurationInfo(tool, gitURL, revision, gh.getGitInfo(targetDir)) + registerConfigurationInfo(meta, tool, gitURL, revision, gh.getGitInfo(targetDir)) } return this diff --git a/meta/folder.groovy b/meta/folder.groovy index d4caa5d..2963e58 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -139,6 +139,31 @@ class ReleaseInfo extends GISJsonSerialised { } } +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String format = '0.1' ProductInfo productinfo @@ -146,7 +171,7 @@ class MetaInfo extends GISJsonSerialised { JenkinsBuildInfo jenkinsbuildinfo BuildStampInfo buildstampinfo ReleaseInfo releaseinfo - + private ArrayList configurationinfo private ArrayList associatedmeta MetaInfo(script) { @@ -156,9 +181,23 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo = new JenkinsBuildInfo(script) buildstampinfo = new BuildStampInfo(script) releaseinfo = new ReleaseInfo(script) + configurationinfo = new ArrayList() associatedmeta = new ArrayList() } + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return associated + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + configurationinfo.add(ci) + } + List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { @@ -179,6 +218,7 @@ class MetaInfo extends GISJsonSerialised { 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), 'associatedmeta' : associatedmetaserialised(), ] return data @@ -189,14 +229,12 @@ class MetaInfo extends GISJsonSerialised { } } -MetaInfo theMetaData = new MetaInfo(this) - -def metareset(script) { +MetaInfo metareset(script) { echo "Resetting meta folder" dir("${WORKSPACE}/${METAFOLDERNAME}") { deleteDir() } - theMetaData = new MetaInfo(this) + return new MetaInfo(script) } return this; \ No newline at end of file diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index b576201..b99467e 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -17,9 +17,6 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() - - def ch = load("${cs}/configuration/configuration.groovy") - ch.saveConfigurationInVersionInfo(); } return this; From 632b08b6e9b7dd4dbf79b36af318428440f5c4ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 20:29:56 +0000 Subject: [PATCH 265/558] Start using meta info for storing configuration --- autobuilder.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index a82e891..e7943c6 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,8 +4,6 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" -metafolder = load("${commonscripts()}/meta/folder.groovy") - def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -28,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -36,6 +34,7 @@ MetaInfo start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONE conditionalclean() + def metafolder = load("${commonscripts()}/meta/folder.groovy") def meta = metafolder.metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) From 97f31cf3137c3db49735435a65175ff250eed030 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 20 Nov 2018 22:52:20 +0000 Subject: [PATCH 266/558] Move functionality to meta --- autobuilder.groovy | 5 ++++- meta/folder.groovy | 13 ++++++++++++- versionnumbering/generate.groovy | 32 +++++++++++++++++++++++++++++--- versionnumbering/versioninfo.py | 25 +++---------------------- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index e7943c6..3a2a408 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -37,7 +37,10 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def metafolder = load("${commonscripts()}/meta/folder.groovy") def meta = metafolder.metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) - + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2963e58..6ae3d3d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -79,7 +79,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { private String buildmachine private String builddate - JenkinsBuildInfo(script, buildjobname = "", buildnumber = 0, buildmachine = "", builddate = "") { + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { super(script) this.buildjobname = buildjobname this.buildnumber = buildnumber @@ -87,6 +87,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } + def set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, @@ -131,6 +138,10 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } + def set(releasejobname) { + this.releasejobname = releasejobname + } + LinkedHashMap serialisationdata() { def data = [ 'releasejobname' : releasejobname, diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index b99467e..0019b24 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -2,16 +2,16 @@ metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" -def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobname = "", versionfolder = "${VERSIONFOLDER}", jobname = "${JOB_NAME}", buildid = "${BUILD_ID}") { +def generate(meta, sourcerevision, sourcepath, basebuildnumber = "1000", versionfolder = "${VERSIONFOLDER}") { def cs = autobuilder.commonscripts() if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" -j "${jobname}" -n "${buildid}" -r "${releasejobname}" + "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" """ } env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() @@ -20,3 +20,29 @@ def generate(sourcerevision, sourcepath, basebuildnumber = "1000", releasejobnam } return this; + + +/* + + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) +*/ diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 370d9fc..0fa097a 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -24,7 +24,6 @@ __date__ = '2017-11-29' __updated__ = '2017-11-29' gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -slaveMachine = os.getenv("NODE_NAME", default="-") MAXVERSIONNUMBER = 65535 def resetFolder(versionfolder): @@ -138,10 +137,7 @@ def getVersionPrefix(sourcebranch): print("Determined version prefix to be: {0}".format(versionprefix)) return versionprefix -def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname): - with open(os.path.join(versionfolder, "buildjobname.txt"), 'w') as f: - f.write(buildjobname) - +def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: f.write(versionprefix) @@ -157,9 +153,6 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: f.write(remoteurl) - with open(os.path.join(versionfolder, "buildnumber.txt"), 'w') as f: - f.write(buildnumber) - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: f.write(buildstamp) @@ -167,15 +160,9 @@ def writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncou with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: f.write(buildstampcommasep) - with open(os.path.join(versionfolder, "buildmachine.txt"), 'w') as f: - f.write(slaveMachine) - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: f.write(str(datetime.datetime.now())) - with open(os.path.join(versionfolder, "releasejobname.txt"), 'w') as f: - f.write(releasejobname) - def main(argv=None): # IGNORE:C0111 '''Command line options.''' @@ -207,9 +194,6 @@ USAGE parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - parser.add_argument("-j", "--buildjobname", dest="buildjobname", action="store", default="-", help="build job name [default: %(default)s]") - parser.add_argument("-n", "--buildnumber", dest="buildnumber", action="store", default="1", help="build number [default: %(default)s]") - parser.add_argument("-r", "--releasejobname", dest="releasejobname", action="store", default="-", help="release-to-git build job name [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -219,12 +203,9 @@ USAGE versionfolder = args.versionfolder gitrootfolder = args.gitrootfolder baseversionnumber = args.baseversionnumber - buildjobname = args.buildjobname - buildnumber = args.buildnumber - releasejobname = args.releasejobname print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber, "Build job name:", buildjobname, "Build number:", buildnumber, "Release job name:", releasejobname) + print("Base version number:", baseversionnumber) resetFolder(versionfolder) @@ -235,7 +216,7 @@ USAGE buildstamp = versionprefix + str(gitrevisioncount) print("Build stamp is: {0}".format(buildstamp)) - writeversionfiles(versionfolder, buildjobname, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildnumber, buildstamp, releasejobname) + writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) return 0 except Exception as e: From d766ef7a14138ebc50170579c86cc6a6ccbce1ae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:08:11 +0000 Subject: [PATCH 267/558] Improve meta gathering --- meta/folder.groovy | 15 ++++ versionnumbering/generate.groovy | 134 +++++++++++++++++++++++-------- versionnumbering/versioninfo.py | 53 ------------ 3 files changed, 114 insertions(+), 88 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6ae3d3d..8b638a9 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -61,6 +61,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + def set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, @@ -119,6 +127,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + def set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 0019b24..f9683e6 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,48 +1,112 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -VERSIONFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/version" +ddef getGitRevision(String gitrootpath, int basebuildnumber) { + dir( gitrootpath) { + def int MAXVERSIONNUMBER = 65535 -def generate(meta, sourcerevision, sourcepath, basebuildnumber = "1000", versionfolder = "${VERSIONFOLDER}") { - def cs = autobuilder.commonscripts() - if (isUnix()) { - sh """ - "${PYTHON}" -u "${cs}/versionnumbering/versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" - """ + def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count") + echo "Change count: ${changecount}" + def int adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + echo "Adjusted change count: ${adjustedchangecount}" + + def gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD") + echo "Revision: ${gitrevision}" + + def remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin") + echo "Remote info: ${remoteinfo}" + + def remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin") + echo "Remote URL: ${remoteurl}" } - else { - bat """ - "${PYTHON}" -u "${cs}\\versionnumbering\\versioninfo.py" -s "${sourcerevision}" -v "${versionfolder}" -g "${sourcepath}" -b "${basebuildnumber}" - """ - } - env.BUILD_STAMP = readFile("${VERSIONFOLDER}/buildstamp.txt").trim() - env.BUILD_STAMP_COMMA_SEP = readFile("${VERSIONFOLDER}/buildstampcommasep.txt").trim() - env.SRC_REVISION = readFile("${VERSIONFOLDER}/revision.txt").trim() + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] } -return this; +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch) { + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + def defaultversionprefix = "1.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } -/* - - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) + def mastertypeversionprefix = "1.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) + echo "sourcebranch is now: ${sourcebranch}" + def splits2 = sourcebranch.split('\\.') + echo "Splits2 is: ${splits2}" + if (splits2.size() < 1) { + def thirdpartyversionprefix = "1.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + splits = [] + for (split in splits2) { + echo "Split is: ${split}" + if (split.isInteger()) { + splits.add(split) + } + else if (split == "x") { + return mastertypeversionprefix + } + } + echo "Splits is: ${splits}" - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) + def versionprefix = "" + if (splits.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int splitnumbercount = 0 + for (split in splits) { + echo "Split is ${split}" + if (split.isInteger()) { + versionprefix += split + "." + splitnumbercount += 1 + } + else if (splitnumbercount > 2) { + break + } + } + if (splitnumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + if (splitnumbercount == 1) { + versionprefix += "0.0." + } + else if (splitnumbercount == 2) { + versionprefix += "0." + } + } + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) + def versionprefix = getVersionPrefix(sourcerevision) - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) -*/ + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + + return buildstamp +} + +return this; \ No newline at end of file diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py index 0fa097a..3d2a80d 100644 --- a/versionnumbering/versioninfo.py +++ b/versionnumbering/versioninfo.py @@ -23,57 +23,6 @@ __version__ = 0.1 __date__ = '2017-11-29' __updated__ = '2017-11-29' -gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") -MAXVERSIONNUMBER = 65535 - -def resetFolder(versionfolder): - print("Removing folders and contents of: {0}".format(versionfolder)) - try: - shutil.rmtree(versionfolder) - except FileNotFoundError: - pass - - print("Creating folder: {0}".format(versionfolder)) - os.makedirs(versionfolder) - -def getGitRevision(gitrootpath, basenumber): - cwd = os.getcwd() - - path = os.path.expanduser(gitrootpath) - print("Changing current working directory to: {0}".format(path)) - os.chdir(path) - - print("Determining revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-list", "HEAD", "--count"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - changecount = completed.stdout.decode("utf-8").strip() - print("Change count: {0}".format(changecount)) - adjustedchangecount = int((int(changecount) + int(basenumber)) % (MAXVERSIONNUMBER + 1)) - print("Adjusted change count: {0}".format(adjustedchangecount)) - - print("Determining current revision info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "rev-parse", "HEAD"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - revision = completed.stdout.decode("utf-8").strip() - print("Revision: {0}".format(revision)) - - print("Determining remote info using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "show", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteinfo = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteinfo)) - - print("Determining remote url using git command: {0}".format(gitExecutable)) - completed = subprocess.run([gitExecutable, "remote", "get-url", "--push", "origin"], stdout=subprocess.PIPE) - print("Command {0} returned: {1}".format(completed.args, completed.returncode)) - remoteurl = completed.stdout.decode("utf-8") - print("Remote info: {0}".format(remoteurl)) - - os.chdir(cwd) - print("Restored current working directory to: {0}".format(cwd)) - - return (adjustedchangecount, revision, remoteinfo, remoteurl) - # Handle: # origin/master # origin/2.0.x @@ -207,8 +156,6 @@ USAGE print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) print("Base version number:", baseversionnumber) - resetFolder(versionfolder) - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) versionprefix = getVersionPrefix(sourcebranch) From 2e674ac1f357abc4b03707963747fa8b7004cdcd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:08:43 +0000 Subject: [PATCH 268/558] Improve meta gathering --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f9683e6..69bdd4d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,5 +1,5 @@ -ddef getGitRevision(String gitrootpath, int basebuildnumber) { +def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 From cd12e5f558506e0ca5517440e1bfcc2a6b7dd14b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:13:47 +0000 Subject: [PATCH 269/558] Improve meta gathering --- meta/folder.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8b638a9..690ffaa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -29,7 +29,7 @@ class ProductInfo extends GISJsonSerialised { this.jirakey = jirakey } - def set(productname, subcomponentname, jirakey) { + void set(String productname, String subcomponentname, String jirakey) { this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jirakey @@ -61,7 +61,7 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } - def set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) { + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { this.sourcepath = sourcepath this.sourcerevision = sourcerevision this.gitrevision = gitrevision @@ -95,7 +95,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } - def set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { this.buildjobname = buildjobname this.buildnumber = buildnumber this.buildmachine = buildmachine @@ -127,7 +127,7 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } - def set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp @@ -153,7 +153,7 @@ class ReleaseInfo extends GISJsonSerialised { this.releasejobname = releasejobname } - def set(releasejobname) { + void set(String releasejobname) { this.releasejobname = releasejobname } @@ -220,7 +220,7 @@ class MetaInfo extends GISJsonSerialised { } void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { - ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) configurationinfo.add(ci) } @@ -263,4 +263,4 @@ MetaInfo metareset(script) { return new MetaInfo(script) } -return this; \ No newline at end of file +return this; From 66cbeb949cfefa8c96c186aa967c8ebf23a2128a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:22:00 +0000 Subject: [PATCH 270/558] Improve meta gathering --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 24 ++++++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 690ffaa..045d67d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -216,7 +216,7 @@ class MetaInfo extends GISJsonSerialised { for (configurationitem in configurationinfo) { configuration.add(configurationitem.serialisationdata()) } - return associated + return configuration } void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 69bdd4d..3c475bc 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,20 +1,24 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count") + def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count").trim() echo "Change count: ${changecount}" - def int adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) echo "Adjusted change count: ${adjustedchangecount}" - def gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD") + gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD").trim() echo "Revision: ${gitrevision}" - def remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin") + remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin").trim() echo "Remote info: ${remoteinfo}" - def remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin") + remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] @@ -41,9 +45,7 @@ def getVersionPrefix(String sourcebranch) { return mastertypeversionprefix } - echo "sourcebranch is now: ${sourcebranch}" def splits2 = sourcebranch.split('\\.') - echo "Splits2 is: ${splits2}" if (splits2.size() < 1) { def thirdpartyversionprefix = "1.25000.0." echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" @@ -51,7 +53,6 @@ def getVersionPrefix(String sourcebranch) { } splits = [] for (split in splits2) { - echo "Split is: ${split}" if (split.isInteger()) { splits.add(split) } @@ -59,7 +60,6 @@ def getVersionPrefix(String sourcebranch) { return mastertypeversionprefix } } - echo "Splits is: ${splits}" def versionprefix = "" if (splits.size() < 1) { @@ -69,7 +69,6 @@ def getVersionPrefix(String sourcebranch) { else { def int splitnumbercount = 0 for (split in splits) { - echo "Split is ${split}" if (split.isInteger()) { versionprefix += split + "." splitnumbercount += 1 @@ -96,6 +95,11 @@ def getVersionPrefix(String sourcebranch) { } String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) def versionprefix = getVersionPrefix(sourcerevision) From 5682c85420b0711b1c1a49c03d535359faf3b5f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:26:40 +0000 Subject: [PATCH 271/558] Improve meta gathering --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 3a2a408..50a1655 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -39,7 +39,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From 1968cf408facba86ee7ab12083953133c54c17cf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:30:56 +0000 Subject: [PATCH 272/558] Improve meta gathering --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 50a1655..bbe7bb9 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -39,7 +39,7 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) - meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID, NODE_NAME, now.toString()) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From fdea15232d3fdab3e7e29fd7c22348acb5d339be Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 10:40:37 +0000 Subject: [PATCH 273/558] Improve meta gathering --- versionnumbering/generate.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 3c475bc..16fb1a9 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -7,18 +7,18 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-list HEAD --count").trim() + def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) echo "Adjusted change count: ${adjustedchangecount}" - gitrevision = sh(returnStdout: true, script: "${GIT_EXECUTABLE} rev-parse HEAD").trim() + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() echo "Revision: ${gitrevision}" - remoteinfo = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote show origin").trim() + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "${GIT_EXECUTABLE} remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 820586ccd2025f28d8b6028c64aa4d6b4bcc6132 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 11:32:04 +0000 Subject: [PATCH 274/558] Improve meta gathering --- differences/do.groovy | 10 ++++------ meta/folder.groovy | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 9e572af..35e0980 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,10 +1,8 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.DIFFERENCEARTIFACTS = "${metafolder.METAFOLDERNAME}/differences/**" -DIFFERENCEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/differences" - -def differences(sourcerevision, previoussourcerevision, sourcepath, jirakey, difffolder = "${DIFFERENCEFOLDER}") { - cs = autobuilder.commonscripts() +def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { + def String cs = autobuilder.commonscripts() + def String difffolder = meta.folder() + def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" diff --git a/meta/folder.groovy b/meta/folder.groovy index 045d67d..25d5f1f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,7 +1,4 @@ -METAFOLDERNAME = "meta" -env.METAARTIFACTS = "${METAFOLDERNAME}/**" - abstract class GISJsonSerialised implements Serializable { def script @@ -20,7 +17,7 @@ abstract class GISJsonSerialised implements Serializable { class ProductInfo extends GISJsonSerialised { private String productname private String subcomponentname - private String jirakey + String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { super(script) @@ -191,6 +188,10 @@ class ConfigurationInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { + private String METAFOLDERNAME = "meta" + private String METAFILENAME = "info.json" + private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private String format = '0.1' ProductInfo productinfo GitInfo gitinfo @@ -250,17 +251,26 @@ class MetaInfo extends GISJsonSerialised { return data } - def setproductinfo(productname, subcomponentname, jirakey) { - productinfo.set(productname, subcomponentname, jirakey) + + String folder() { + return METAFOLDERNAME + } + + String generateartifacts() { + def String metafilepath = "${METAFOLDERNAME}/${METAFILENAME}" + saveJsonSerialised(metafilepath) + return METAARTIFACTS } } MetaInfo metareset(script) { echo "Resetting meta folder" - dir("${WORKSPACE}/${METAFOLDERNAME}") { + def mi = new MetaInfo(script) + + dir(mi.folder()) { deleteDir() } - return new MetaInfo(script) + return mi } return this; From eee94ee26282f8b204e742a5b14c64997cc52c7a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 13:15:10 +0000 Subject: [PATCH 275/558] Add checksum info to meta --- checksum/utils.groovy | 15 +++----- meta/folder.groovy | 83 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 11 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index 748472f..c090bf3 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -1,21 +1,14 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.CHECKSUMARTIFACTS = "${metafolder.METAFOLDERNAME}/checksums/**" -CHECKSUMFOLDER = "${metafolder.METAFOLDERNAME}/checksums" - // Generate a checksum file of from a list of files/directories relative to workspace -def generatelist(checksummable) { - load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - sums = "" +def generatelist(meta, checksummable) { for (filepath in checksummable) { - echo "Checksumming ${filepath}" + def checksumset = meta.checksuminfo.addchecksumset(filepath) def files = findFiles(glob: filepath) for (file in files) { - def sum = sha1(file: file.path) - sums += "${sum} ${file.path}${nl}" // Format suitable for sha1sum utility + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) } } - writeFile(file: "${CHECKSUMFOLDER}/sha1sumlist.txt", text: sums) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 25d5f1f..152121c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -187,6 +187,86 @@ class ConfigurationInfo extends GISJsonSerialised { } } +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath, String sum) { + super(checksumset.script) + this.filepath = filepath + this.sum = sum + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String artifactpath + private ArrayList checksumpairs + + ChecksumInfo(script, artifactpath) { + super(script) + this.artifactpath = artifactpath + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } + + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" @@ -198,6 +278,7 @@ class MetaInfo extends GISJsonSerialised { JenkinsBuildInfo jenkinsbuildinfo BuildStampInfo buildstampinfo ReleaseInfo releaseinfo + ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList associatedmeta @@ -208,6 +289,7 @@ class MetaInfo extends GISJsonSerialised { jenkinsbuildinfo = new JenkinsBuildInfo(script) buildstampinfo = new BuildStampInfo(script) releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() associatedmeta = new ArrayList() } @@ -246,6 +328,7 @@ class MetaInfo extends GISJsonSerialised { 'buildstampinfo' : buildstampinfo.serialisationdata(), 'releaseinfo' : releaseinfo.serialisationdata(), 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), 'associatedmeta' : associatedmetaserialised(), ] return data From 1eaf0617311f9b3608d258a6f4eebc16392ecc7b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 18:33:55 +0000 Subject: [PATCH 276/558] Add checksum info to meta --- meta/folder.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 152121c..9a89b70 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -192,7 +192,7 @@ class ChecksumPair extends GISJsonSerialised { private String sum ChecksumPair(script, String filepath, String sum) { - super(checksumset.script) + super(script) this.filepath = filepath this.sum = sum } @@ -210,9 +210,10 @@ class ChecksumSet extends GISJsonSerialised { private String artifactpath private ArrayList checksumpairs - ChecksumInfo(script, artifactpath) { + ChecksumSet(script, artifactpath) { super(script) this.artifactpath = artifactpath + checksumpairs = new ArrayList() } ChecksumPair addChecksumPair(String filepath, String sum) { @@ -236,17 +237,20 @@ class ChecksumSet extends GISJsonSerialised { ] return data } - +} class ChecksumInfo extends GISJsonSerialised { private ArrayList checksumsets ChecksumInfo(script) { super(script) + checksumsets = new ArrayList() } ChecksumSet addChecksumSet(String artifactpath) { + script.echo "Entering addChecksumSet" def cs = new ChecksumSet(script, artifactpath) + script.echo "Created ChecksumSet" checksumsets.add(cs) return cs } @@ -339,8 +343,7 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts() { - def String metafilepath = "${METAFOLDERNAME}/${METAFILENAME}" + String generateartifacts(String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}") { saveJsonSerialised(metafilepath) return METAARTIFACTS } From e7fb9a8805581a27fd54425d85994d1911969e5d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 18:40:12 +0000 Subject: [PATCH 277/558] Add checksum info to meta --- checksum/utils.groovy | 4 ++-- meta/folder.groovy | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/checksum/utils.groovy b/checksum/utils.groovy index c090bf3..c2503cd 100644 --- a/checksum/utils.groovy +++ b/checksum/utils.groovy @@ -2,7 +2,7 @@ // Generate a checksum file of from a list of files/directories relative to workspace def generatelist(meta, checksummable) { for (filepath in checksummable) { - def checksumset = meta.checksuminfo.addchecksumset(filepath) + def checksumset = meta.checksuminfo.addChecksumSet(filepath) def files = findFiles(glob: filepath) for (file in files) { def sum = sha1(file: file.path).trim() @@ -11,4 +11,4 @@ def generatelist(meta, checksummable) { } } -return this; +return this; \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy index 9a89b70..421fec8 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -248,9 +248,7 @@ class ChecksumInfo extends GISJsonSerialised { } ChecksumSet addChecksumSet(String artifactpath) { - script.echo "Entering addChecksumSet" def cs = new ChecksumSet(script, artifactpath) - script.echo "Created ChecksumSet" checksumsets.add(cs) return cs } From d44af429e0ce3720419f221240d116601b87a4c1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 20:19:08 +0000 Subject: [PATCH 278/558] Add publishable items to meta --- meta/folder.groovy | 32 ++++++++++++++++++++++++++++++++ publishing/utils.groovy | 20 ++------------------ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 421fec8..7471f74 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -269,6 +269,22 @@ class ChecksumInfo extends GISJsonSerialised { } } +class PublishableInfo extends GISJsonSerialised { + private String filepath + + PublishableInfo(script, String filepath = "") { + super(script) + this.filepath = filepath + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + ] + return data + } +} + class MetaInfo extends GISJsonSerialised { private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" @@ -282,6 +298,7 @@ class MetaInfo extends GISJsonSerialised { ReleaseInfo releaseinfo ChecksumInfo checksuminfo private ArrayList configurationinfo + private ArrayList publishableinfo private ArrayList associatedmeta MetaInfo(script) { @@ -293,6 +310,7 @@ class MetaInfo extends GISJsonSerialised { releaseinfo = new ReleaseInfo(script) checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() + publishableinfo = new ArrayList() associatedmeta = new ArrayList() } @@ -304,11 +322,24 @@ class MetaInfo extends GISJsonSerialised { return configuration } + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + publishableinfo.add(pi) + } + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) configurationinfo.add(ci) } + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + List associatedmetaserialised() { def associated = [] for (metaitem in associatedmeta) { @@ -331,6 +362,7 @@ class MetaInfo extends GISJsonSerialised { 'releaseinfo' : releaseinfo.serialisationdata(), 'configuration' : configurationinfoserialised(), 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), 'associatedmeta' : associatedmetaserialised(), ] return data diff --git a/publishing/utils.groovy b/publishing/utils.groovy index 72c6566..145a7e9 100644 --- a/publishing/utils.groovy +++ b/publishing/utils.groovy @@ -1,26 +1,10 @@ -metafolder = load("${autobuilder.commonscripts()}/meta/folder.groovy") -env.PUBLISHABLEARTIFACTS = "${metafolder.METAFOLDERNAME}/publishable/**" - // Generate a list of artifact files that are publishable to outside parties -def generatelist(publishable) { - def PUBLISHABLEFOLDER = "${WORKSPACE}/${metafolder.METAFOLDERNAME}/publishable" - sh """ - rm -rf "${PUBLISHABLEFOLDER}" - mkdir -p "${PUBLISHABLEFOLDER}" - """ +def generatelist(meta, publishable) { filelist = [] for (filepath in publishable) { - def files = findFiles(glob: filepath) - for (file in files) { - if (!file.directory) { - filelist.add(file.path) - } - } + meta.addPublishable(filepath) } - def outJson = readJSON text: groovy.json.JsonOutput.toJson(filelist) - def PUBLISHABLEFILE = "${PUBLISHABLEFOLDER}/list.json" - writeJSON(file: PUBLISHABLEFILE, json: outJson, pretty: 4) } return this; From e5cb43356948c2cb30e13289c80b2d08190c0c12 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Nov 2018 22:17:38 +0000 Subject: [PATCH 279/558] Add release information items to meta --- meta/folder.groovy | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7471f74..b5cc379 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -143,20 +143,35 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private String releasejobname + private String releaserepoURL + private ArrayList releaseitems - ReleaseInfo(script, releasejobname = "") { + ReleaseInfo(script, releaserepoURL = "") { super(script) - this.releasejobname = releasejobname + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() } - void set(String releasejobname) { - this.releasejobname = releasejobname + void set(String releaserepoURL = "") { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + releaseitems.add(releaseitem) + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items } LinkedHashMap serialisationdata() { def data = [ - 'releasejobname' : releasejobname, + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), ] return data } @@ -373,9 +388,21 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - String generateartifacts(String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}") { + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), + ArrayList releasedfromsubmeta = new ArrayList(), + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = METAARTIFACTS + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } saveJsonSerialised(metafilepath) - return METAARTIFACTS + return artifactlist } } From 6adf4cd514083b8e249e5d86a23e88470c608267 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 17:23:17 +0000 Subject: [PATCH 280/558] Deserialise meta from file and associate with job meta --- import/importer.groovy | 14 ++++ meta/folder.groovy | 154 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 import/importer.groovy diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..ba912f4 --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,14 @@ + +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: buildselector + ], + target: destpath + ) + meta.importrelated(destpath) +} \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy index b5cc379..9d90cee 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -6,7 +6,9 @@ abstract class GISJsonSerialised implements Serializable { this.script = script } - abstract LinkedHashMap serialisationdata(); + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) def saveJsonSerialised(String outputfile, int prettylevel = 4) { def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) @@ -32,6 +34,12 @@ class ProductInfo extends GISJsonSerialised { this.jirakey = jirakey } + void deserialiseJson(rawmeta) { + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + jirakey = rawmeta['jirakey'] + } + LinkedHashMap serialisationdata() { def data = [ 'productname' : productname, @@ -66,6 +74,14 @@ class GitInfo extends GISJsonSerialised { this.remoteinfo = remoteinfo } + void deserialiseJson(rawmeta) { + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + LinkedHashMap serialisationdata() { def data = [ 'sourcepath' : sourcepath, @@ -99,6 +115,13 @@ class JenkinsBuildInfo extends GISJsonSerialised { this.builddate = builddate } + void deserialiseJson(rawmeta) { + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + } + LinkedHashMap serialisationdata() { def data = [ 'buildjobname' : buildjobname, @@ -131,6 +154,13 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + void deserialiseJson(rawmeta) { + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + buildstampcommasep = rawmeta['buildstampcommasep'] + } + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, @@ -168,6 +198,14 @@ class ReleaseInfo extends GISJsonSerialised { return items } + void deserialiseJson(rawmeta) { + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + } + LinkedHashMap serialisationdata() { def data = [ 'releaserepoURL' : releaserepoURL, @@ -191,6 +229,13 @@ class ConfigurationInfo extends GISJsonSerialised { this.configurationSHA = configurationSHA } + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + LinkedHashMap serialisationdata() { def data = [ 'configurationname' : configurationname, @@ -212,6 +257,11 @@ class ChecksumPair extends GISJsonSerialised { this.sum = sum } + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + LinkedHashMap serialisationdata() { def data = [ 'filepath' : filepath, @@ -245,6 +295,16 @@ class ChecksumSet extends GISJsonSerialised { return associated } + void deserialiseJson(rawmeta) { + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + LinkedHashMap serialisationdata() { def data = [ 'artifactpath' : artifactpath, @@ -276,6 +336,15 @@ class ChecksumInfo extends GISJsonSerialised { return associated } + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + LinkedHashMap serialisationdata() { def data = [ 'checksumsets' : checksumsetsserialised(), @@ -292,6 +361,10 @@ class PublishableInfo extends GISJsonSerialised { this.filepath = filepath } + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + } + LinkedHashMap serialisationdata() { def data = [ 'filepath' : filepath, @@ -301,11 +374,37 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { + private String FORMAT = "0.1" private String METAFOLDERNAME = "meta" private String METAFILENAME = "info.json" private String METAARTIFACTS = "${METAFOLDERNAME}/**" - private String format = '0.1' + class RelatedMetaInfo extends GISJsonSerialised { + String offsetpath + MetaInfo meta + + RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + super(script) + this.offsetpath = offsetpath + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'offsetpath' : offsetpath, + 'meta' : meta.serialisationdata(), + ] + return data + } + } + + private String format = FORMAT ProductInfo productinfo GitInfo gitinfo JenkinsBuildInfo jenkinsbuildinfo @@ -314,7 +413,7 @@ class MetaInfo extends GISJsonSerialised { ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo - private ArrayList associatedmeta + private ArrayList associatedmeta MetaInfo(script) { super(script) @@ -326,7 +425,7 @@ class MetaInfo extends GISJsonSerialised { checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() publishableinfo = new ArrayList() - associatedmeta = new ArrayList() + associatedmeta = new ArrayList() } List configurationinfoserialised() { @@ -363,8 +462,51 @@ class MetaInfo extends GISJsonSerialised { return associated } - void addAssociatedMetaInfo(mi) { - associatedmeta.add(mi) + void deserialiseJson(rawmeta) { + if (rawmeta['format'] != FORMAT) { + script.error("Unsupported meta format found at ${filepath}") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + for (rawrelatedmeta in rawmeta['associatedmeta']) { + def rm = new RelatedMetaInfo(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + + def deserialiseJson(String filepath) { + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + MetaInfo importrelated(String destpath) { + MetaInfo mi = new MetaInfo(script) + + def importedmetafile = destpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + } + + RelatedMetaInfo rmi = new RelatedMetaInfo(script, destpath, mi) + associatedmeta.add(rmi) + + return mi } LinkedHashMap serialisationdata() { From 214f2db8da535ed2cc88562a995e922343ccf837 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 18:51:51 +0000 Subject: [PATCH 281/558] Deserialise meta from file and associate with job meta --- import/importer.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index ba912f4..d3c97e4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -11,4 +11,6 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) meta.importrelated(destpath) -} \ No newline at end of file +} + +return this; \ No newline at end of file From 5d333d926dd73727555dfbad94768e9d91a26b69 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 19:54:32 +0000 Subject: [PATCH 282/558] Store offset paths relative to WORKSPACE if given as absolute paths --- meta/folder.groovy | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9d90cee..28fe85f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,3 +1,5 @@ +import java.nio.file.Paths +import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { def script @@ -494,16 +496,27 @@ class MetaInfo extends GISJsonSerialised { def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } + + String relativize(String path) { + def Path thispath = Paths.get(path) + if (thispath.startsWith(script.WORKSPACE)) { + def Path wspath = Paths.get(script.WORKSPACE) + def Path wsrelativePath = wspath.relativize(thispath) + return wsrelativePath.toString() + } + return path + } MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) - def importedmetafile = destpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + METAFOLDERNAME + "/" + METAFILENAME if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) } - RelatedMetaInfo rmi = new RelatedMetaInfo(script, destpath, mi) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) return mi From 61b8d45657d6d84ddcb853c8af90817810ff732b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 20:06:10 +0000 Subject: [PATCH 283/558] Only show SHA code of configuration --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 031094b..cb9354b 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -2,7 +2,7 @@ clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { dir(workingSetRootPath) { - return clh.getCommandOutput('git log -1 --format=oneline') + return clh.getCommandOutput('git rev-parse HEAD') } } From 31918ae70080d0ba226f2896362fb2f73c176e54 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 21:05:06 +0000 Subject: [PATCH 284/558] Handle more version number strings --- versionnumbering/generate.groovy | 46 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 16fb1a9..59fe2a3 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -28,6 +28,8 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { + def int MAXVERSIONNUMBER = 65535 + echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -51,43 +53,45 @@ def getVersionPrefix(String sourcebranch) { echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" return thirdpartyversionprefix } - splits = [] + for (split in splits2) { - if (split.isInteger()) { - splits.add(split) - } - else if (split == "x") { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" return mastertypeversionprefix } } + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + int integer = integerstring.toInteger() + integer = integer % (MAXVERSIONNUMBER + 1) + integers.add(integer) + } + def versionprefix = "" - if (splits.size() < 1) { + if (integers.size() < 1) { echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" versionprefix = defaultversionprefix } else { - def int splitnumbercount = 0 - for (split in splits) { - if (split.isInteger()) { - versionprefix += split + "." - splitnumbercount += 1 - } - else if (splitnumbercount > 2) { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { break } } - if (splitnumbercount == 0) { + if (integernumbercount == 0) { echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" versionprefix = defaultversionprefix } - else { - if (splitnumbercount == 1) { - versionprefix += "0.0." - } - else if (splitnumbercount == 2) { - versionprefix += "0." - } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." } } echo "Determined version prefix to be: ${versionprefix}" From 9bd5d03e498beec93f8caba72ff45753d7595e3f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 21:36:36 +0000 Subject: [PATCH 285/558] Parameterise the release repo --- autobuilder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index bbe7bb9..732e268 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -26,7 +26,7 @@ def getautobuildpath() { } // Start up boilderplate -def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath @@ -40,7 +40,9 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - + + meta.releaseinfo.set(releaserepo) + def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) From ce1ac15aac688b102ec175f8bab6714555da7db1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:20:04 +0000 Subject: [PATCH 286/558] Check for existence of release repo by checking it out --- autobuilder.groovy | 3 ++- release/registerrepo.groovy | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 release/registerrepo.groovy diff --git a/autobuilder.groovy b/autobuilder.groovy index 732e268..2a04925 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -41,7 +41,8 @@ def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NA now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) - meta.releaseinfo.set(releaserepo) + def rr = load("${commonscripts()}/release/registerrepo.groovy") + rr.registerrepo(meta, releaserepo) def ch = load("${commonscripts()}/configuration/configuration.groovy") ch.registerPipelineScriptConfiguration(meta) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..1fc76ab --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo) { + if (releaserepo?.trim()) { + vc.gitCheckout(releaserepo, "origin/master", "meta/releaserepo") + } + meta.releaseinfo.set(releaserepo) +} + +return this; From dcbf2c6e116c256dc7a66f76ab9195d7b9836ba5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:32:50 +0000 Subject: [PATCH 287/558] Save the temp repo load in a tmp folder --- release/registerrepo.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy index 1fc76ab..ff4d85d 100644 --- a/release/registerrepo.groovy +++ b/release/registerrepo.groovy @@ -4,7 +4,7 @@ vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") // Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data def registerrepo(meta, String releaserepo) { if (releaserepo?.trim()) { - vc.gitCheckout(releaserepo, "origin/master", "meta/releaserepo") + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") } meta.releaseinfo.set(releaserepo) } From 59afde3c0f7ce382c624e44768bb7ff80823302b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Nov 2018 22:59:38 +0000 Subject: [PATCH 288/558] Make compile job use meta --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 28fe85f..0f9e0d5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -138,8 +138,8 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix private int revision - private String buildstamp - private String buildstampcommasep + String buildstamp + String buildstampcommasep BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { super(script) From c6268663842da55ff11e820aa738246e5a1d1e91 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 11:24:37 +0000 Subject: [PATCH 289/558] Rename compile_init --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0f9e0d5..189014c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -162,7 +162,7 @@ class BuildStampInfo extends GISJsonSerialised { buildstamp = rawmeta['buildstamp'] buildstampcommasep = rawmeta['buildstampcommasep'] } - + LinkedHashMap serialisationdata() { def data = [ 'buildstampprefix' : buildstampprefix, From 78d78c5f8fc2d4a331e0e30752b25244339a9e1c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 12:34:53 +0000 Subject: [PATCH 290/558] Correct checksumset deserialisation --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 189014c..e9491fe 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -253,7 +253,7 @@ class ChecksumPair extends GISJsonSerialised { private String filepath private String sum - ChecksumPair(script, String filepath, String sum) { + ChecksumPair(script, String filepath = "", String sum = "") { super(script) this.filepath = filepath this.sum = sum @@ -277,7 +277,7 @@ class ChecksumSet extends GISJsonSerialised { private String artifactpath private ArrayList checksumpairs - ChecksumSet(script, artifactpath) { + ChecksumSet(script, artifactpath = "") { super(script) this.artifactpath = artifactpath checksumpairs = new ArrayList() From fd572074f1916cdade51a53e4d93fcdb735deb20 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 13:18:35 +0000 Subject: [PATCH 291/558] Move image create job to use meta --- meta/folder.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index e9491fe..8f26265 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -507,11 +507,15 @@ class MetaInfo extends GISJsonSerialised { return path } + String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) def String adjustedpath = relativize(destpath) - def importedmetafile = adjustedpath + "/" + METAFOLDERNAME + "/" + METAFILENAME + def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) } From 0e31f84cf9f589fac5b14a398a03b45ad617ed96 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 24 Nov 2018 14:14:13 +0000 Subject: [PATCH 292/558] Update build job to meta --- artifacts/metaexport.groovy | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 66a8153..0cfc773 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,21 +1,19 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders -def combineSubjobArtifacts(subJobs) { - artifactFolders = "" - for (subJob in subJobs) { - subJobObject = subJob[0] - artifactFolder = subJob[1] - if (subJobObject != null) { - subJobName = subJobObject.getProjectName() - subJobNumber = subJobObject.getNumber() - destFolder = "${artifactFolder}" +def combineSubjobArtifacts(meta, subJobs) { + def im = load("${commonscripts()}/import/importer.groovy") - dir("${destFolder}") { + def artifactFolders = "" + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getProjectName() + def subJobNumber = subJobObject.getNumber() + destFolder = subJob[1] + dir(destFolder) { deleteDir() } - - echo "Copying artifacts from: Job: ${subJobName} Number: ${subJobNumber} to folder: ${destFolder}" - copyArtifacts fingerprintArtifacts: true, projectName: "${subJobName}", selector: specific("${subJobNumber}"), target: "${destFolder}" + im.artifactimport(meta, subJobName, specific(subJobNumber), destFolder) artifactFolders += "${destFolder}/**," } } From 8522eb079ab4171be303505102dfaa7551fdc92c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 25 Nov 2018 17:01:32 +0000 Subject: [PATCH 293/558] Update build job to meta --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0cfc773..016add9 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,7 +1,7 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, subJobs) { - def im = load("${commonscripts()}/import/importer.groovy") + def im = load("${autobuilder.commonscripts()}/import/importer.groovy") def artifactFolders = "" for (subJob in subJobs) { From 59d7b3f00f8433982a621d918b696c0d889abeaa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 25 Nov 2018 23:19:27 +0000 Subject: [PATCH 294/558] Make release job use meta --- artifacts/metaexport.groovy | 4 ++-- import/importer.groovy | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 016add9..62a6ba4 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -1,7 +1,7 @@ // Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders def combineSubjobArtifacts(meta, subJobs) { - def im = load("${autobuilder.commonscripts()}/import/importer.groovy") + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def artifactFolders = "" for (subJob in subJobs) { @@ -13,7 +13,7 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - im.artifactimport(meta, subJobName, specific(subJobNumber), destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders += "${destFolder}/**," } } diff --git a/import/importer.groovy b/import/importer.groovy index d3c97e4..a9e28b2 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,15 @@ +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: specific(buildnumber.toString()), + target: destpath + ) + meta.importrelated(destpath) +} + def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -13,4 +24,4 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat meta.importrelated(destpath) } -return this; \ No newline at end of file +return this; From 314da99324a31ab00a74d0e723304d483f55f222 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 08:29:56 +0000 Subject: [PATCH 295/558] Build up list of artifacts --- artifacts/metaexport.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 62a6ba4..cf5e9b3 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -3,7 +3,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def artifactFolders = "" + def artifactFolders = [] for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,7 +14,7 @@ def combineSubjobArtifacts(meta, subJobs) { deleteDir() } importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - artifactFolders += "${destFolder}/**," + artifactFolders.add("${destFolder}/**") } } return artifactFolders; From eb1bef6b699296f5e027c71f8e53bbbcfbec5421 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 11:16:53 +0000 Subject: [PATCH 296/558] Explicitly return artifact list as ArrayList --- artifacts/metaexport.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index cf5e9b3..b6a832b 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -3,7 +3,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - def artifactFolders = [] + def ArrayList artifactFolders = new ArrayList() for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { From 85de3c5974d87ef1843df4cbc8275aea08f10087 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 18:17:55 +0000 Subject: [PATCH 297/558] Walk meta data for release info --- import/importer.groovy | 4 ++-- meta/folder.groovy | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index a9e28b2..4d24d30 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -7,7 +7,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +21,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f26265..c2c30a4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -175,8 +175,8 @@ class BuildStampInfo extends GISJsonSerialised { } class ReleaseInfo extends GISJsonSerialised { - private String releaserepoURL - private ArrayList releaseitems + String releaserepoURL + ArrayList releaseitems ReleaseInfo(script, releaserepoURL = "") { super(script) @@ -563,6 +563,26 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } + + // Recurse tree of meta looking for release info and accumulate path + private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { + if (releaseinfo.releaserepoURL) { + ArrayList rmi(script, pathsofar, this) + releasemetacollection.add(rmi) + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath + releasemetacollection = releasewalkerp(releasemetacollection, newpathsofar) + } + return releasemetacollection + } + + ArrayList releasewalker(String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(releasemetacollection, startpath) + return releasemetacollection + } } MetaInfo metareset(script) { From 1ed79e56b659234eab8991dbb08fcd742bf434a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 18:37:54 +0000 Subject: [PATCH 298/558] Walk meta data for release info --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index c2c30a4..361c120 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -567,7 +567,7 @@ class MetaInfo extends GISJsonSerialised { // Recurse tree of meta looking for release info and accumulate path private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { if (releaseinfo.releaserepoURL) { - ArrayList rmi(script, pathsofar, this) + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) releasemetacollection.add(rmi) } for (relatedmetainfo in associatedmeta) { From 011c5393b28a5897093341dd67c411f438a8d8ab Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 26 Nov 2018 21:18:12 +0000 Subject: [PATCH 299/558] Walk meta data for release info --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 361c120..b204462 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -573,7 +573,7 @@ class MetaInfo extends GISJsonSerialised { for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath - releasemetacollection = releasewalkerp(releasemetacollection, newpathsofar) + releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } return releasemetacollection } From 5505e5e7ba315787bfcd78dfbe7567dff0214ae7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 08:34:19 +0000 Subject: [PATCH 300/558] Correct release path --- meta/folder.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index b204462..88c5e17 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -572,7 +572,10 @@ class MetaInfo extends GISJsonSerialised { } for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + "/" + relatedmetainfo.offsetpath + if (pathsofar) { + pathsofar += "/" + } + String newpathsofar = pathsofar + relatedmetainfo.offsetpath releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } return releasemetacollection From 66eec970b67bb0667feb4b85d0b9caa94947f4ba Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 08:51:27 +0000 Subject: [PATCH 301/558] Correct release path --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 88c5e17..802443d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -570,11 +570,11 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) releasemetacollection.add(rmi) } + if (pathsofar) { + pathsofar += "/" + } for (relatedmetainfo in associatedmeta) { MetaInfo othermeta = relatedmetainfo.meta - if (pathsofar) { - pathsofar += "/" - } String newpathsofar = pathsofar + relatedmetainfo.offsetpath releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) } From 57a8b9bf3714d3f3bd33dfc4faa6608f695958be Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 16:52:38 +0000 Subject: [PATCH 302/558] Check out release repos --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 802443d..f5e7c93 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,7 +564,7 @@ class MetaInfo extends GISJsonSerialised { return artifactlist } - // Recurse tree of meta looking for release info and accumulate path + // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { if (releaseinfo.releaserepoURL) { RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) From 6aa746b16c5a090a5f57a282a5dc136970bcf8e0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 18:09:27 +0000 Subject: [PATCH 303/558] Check out release repos --- meta/folder.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f5e7c93..1e45319 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -507,7 +507,7 @@ class MetaInfo extends GISJsonSerialised { return path } - String metafilepath() { + static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } @@ -598,4 +598,9 @@ MetaInfo metareset(script) { return mi } +MetaInfo metafromfolder(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") +} + return this; From 75882404038a9540dd29ae3984fd352d62074b73 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 18:47:42 +0000 Subject: [PATCH 304/558] Check out release repos --- meta/folder.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1e45319..de445f1 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -376,10 +376,10 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private String FORMAT = "0.1" - private String METAFOLDERNAME = "meta" - private String METAFILENAME = "info.json" - private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String FORMAT = "0.1" + private static String METAFOLDERNAME = "meta" + private static String METAFILENAME = "info.json" + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath From fda8b6fdff44bef328f8f5d2f49b0eca0e39c152 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Tue, 27 Nov 2018 18:55:45 +0000 Subject: [PATCH 305/558] ELGIP-20 moved common_scripts directory as this seems to be expected now --- .project | 17 + .pydevproject | 8 + .settings/org.eclipse.core.resources.prefs | 2 + artifacts/metaexport.groovy | 23 + autobuilder.groovy | 60 ++ checksum/utils.groovy | 14 + clean/clean.groovy | 10 + commandline/commandline.groovy | 13 + .../eclipsebuild/disableparallelbuilds.groovy | 12 + compile/eclipsebuild/eclipseds5build.groovy | 198 ++++++ configuration/configuration.groovy | 16 + differences/.project | 17 + differences/.pydevproject | 8 + differences/do.groovy | 18 + differences/json.groovy | 15 + differences/show_release_differences.py | 226 +++++++ file/utils.groovy | 16 + import/importer.groovy | 27 + install/installhelper.groovy | 42 ++ meta/folder.groovy | 578 ++++++++++++++++++ platform/lineendings.groovy | 5 + publishing/utils.groovy | 10 + release/registerrepo.groovy | 12 + versioncontrol/git.groovy | 67 ++ versionnumbering/do.groovy | 20 + versionnumbering/generate.groovy | 120 ++++ versionnumbering/metaversion.groovy | 100 +++ versionnumbering/versioninfo.py | 180 ++++++ versionstamping/.project | 17 + versionstamping/.pydevproject | 8 + versionstamping/versionstamper.py | 41 ++ 31 files changed, 1900 insertions(+) create mode 100644 .project create mode 100644 .pydevproject create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 artifacts/metaexport.groovy create mode 100644 autobuilder.groovy create mode 100644 checksum/utils.groovy create mode 100644 clean/clean.groovy create mode 100644 commandline/commandline.groovy create mode 100644 compile/eclipsebuild/disableparallelbuilds.groovy create mode 100644 compile/eclipsebuild/eclipseds5build.groovy create mode 100644 configuration/configuration.groovy create mode 100644 differences/.project create mode 100644 differences/.pydevproject create mode 100644 differences/do.groovy create mode 100644 differences/json.groovy create mode 100644 differences/show_release_differences.py create mode 100644 file/utils.groovy create mode 100644 import/importer.groovy create mode 100644 install/installhelper.groovy create mode 100644 meta/folder.groovy create mode 100644 platform/lineendings.groovy create mode 100644 publishing/utils.groovy create mode 100644 release/registerrepo.groovy create mode 100644 versioncontrol/git.groovy create mode 100644 versionnumbering/do.groovy create mode 100644 versionnumbering/generate.groovy create mode 100644 versionnumbering/metaversion.groovy create mode 100644 versionnumbering/versioninfo.py create mode 100644 versionstamping/.project create mode 100644 versionstamping/.pydevproject create mode 100644 versionstamping/versionstamper.py diff --git a/.project b/.project new file mode 100644 index 0000000..e329c3f --- /dev/null +++ b/.project @@ -0,0 +1,17 @@ + + + Common Build Scripts + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/.pydevproject b/.pydevproject new file mode 100644 index 0000000..1ce5929 --- /dev/null +++ b/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 2.7 +python35 + diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 0000000..1174a06 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding//versionnumbering/versioninfo.py=utf-8 diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy new file mode 100644 index 0000000..b6a832b --- /dev/null +++ b/artifacts/metaexport.groovy @@ -0,0 +1,23 @@ + +// Take a list of Sub-job descriptors and output folder pairs, copy the artifacts, while filling in a string of artifact folders +def combineSubjobArtifacts(meta, subJobs) { + def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + + def ArrayList artifactFolders = new ArrayList() + for (subJob in subJobs) { + def subJobObject = subJob[0] + if (subJobObject != null) { + def subJobName = subJobObject.getProjectName() + def subJobNumber = subJobObject.getNumber() + destFolder = subJob[1] + dir(destFolder) { + deleteDir() + } + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + artifactFolders.add("${destFolder}/**") + } + } + return artifactFolders; +} + +return this; diff --git a/autobuilder.groovy b/autobuilder.groovy new file mode 100644 index 0000000..2a04925 --- /dev/null +++ b/autobuilder.groovy @@ -0,0 +1,60 @@ +// Assume that WORKSPACE environment variable is set + +autobuildroot = "${WORKSPACE}/autobuild" + +commonscriptsroot = "${autobuildroot}/Common_scripts" + +def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { + if (doclean) { + echo "Cleaning workspace" + // Save any script files that were loaded into the workspace at first and are required after cleaning the workspace + stash allowEmpty: true, includes: 'autobuild/**', name: 'Clean workspace stash', useDefaultExcludes: false + cleanWs() + unstash name: 'Clean workspace stash' + } + else { + echo "Not cleaning workspace" + } +} + +def commonscripts() { + return commonscriptsroot +} + +def getautobuildpath() { + return autobuildroot +} + +// Start up boilderplate +def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { + echo "Starting...." + + autobuildroot = autobuildpath + commonscriptsroot = commonscriptspath + + conditionalclean() + + def metafolder = load("${commonscripts()}/meta/folder.groovy") + def meta = metafolder.metareset(script) + meta.productinfo.set(productname, subcomponentname, jirakey) + def now = new Date() + now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) + meta.jenkinsbuildinfo.set(JOB_NAME, BUILD_ID.toInteger(), NODE_NAME, now.toString()) + + def rr = load("${commonscripts()}/release/registerrepo.groovy") + rr.registerrepo(meta, releaserepo) + + def ch = load("${commonscripts()}/configuration/configuration.groovy") + ch.registerPipelineScriptConfiguration(meta) + + return meta +} + +def configurationhelper() { + return load("${commonscripts()}/configuration/configuration.groovy") +} + +// Everyone wants to know this +autobuilder = this + +return this; diff --git a/checksum/utils.groovy b/checksum/utils.groovy new file mode 100644 index 0000000..c2503cd --- /dev/null +++ b/checksum/utils.groovy @@ -0,0 +1,14 @@ + +// Generate a checksum file of from a list of files/directories relative to workspace +def generatelist(meta, checksummable) { + for (filepath in checksummable) { + def checksumset = meta.checksuminfo.addChecksumSet(filepath) + def files = findFiles(glob: filepath) + for (file in files) { + def sum = sha1(file: file.path).trim() + checksumset.addChecksumPair(file.path, sum) + } + } +} + +return this; \ No newline at end of file diff --git a/clean/clean.groovy b/clean/clean.groovy new file mode 100644 index 0000000..4707ba0 --- /dev/null +++ b/clean/clean.groovy @@ -0,0 +1,10 @@ + +def clean(folderlist) { + for (folder in folderlist) { + dir(folder) { + deleteDir() + } + } +} + +return this; \ No newline at end of file diff --git a/commandline/commandline.groovy b/commandline/commandline.groovy new file mode 100644 index 0000000..df99a75 --- /dev/null +++ b/commandline/commandline.groovy @@ -0,0 +1,13 @@ +String getCommandOutput(String cmd) { + if (isUnix()) { + return sh(returnStdout: true, script: '#!/bin/sh -e\n' + cmd).trim() + } else { + return bat(returnStdout: true, script: """ + @echo off + ${cmd} + """ + ).trim() + } +} + +return this \ No newline at end of file diff --git a/compile/eclipsebuild/disableparallelbuilds.groovy b/compile/eclipsebuild/disableparallelbuilds.groovy new file mode 100644 index 0000000..03e4c8f --- /dev/null +++ b/compile/eclipsebuild/disableparallelbuilds.groovy @@ -0,0 +1,12 @@ + +// Disable parallel make builds - which may be causing autobuild issues +def doDisableParallelMake(rootSearchFolder = "${WORKSPACE}") { + if (isUnix()) { + sh """ + # Looks for " + + Release_difference + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/differences/.pydevproject b/differences/.pydevproject new file mode 100644 index 0000000..c3434da --- /dev/null +++ b/differences/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +Default + diff --git a/differences/do.groovy b/differences/do.groovy new file mode 100644 index 0000000..35e0980 --- /dev/null +++ b/differences/do.groovy @@ -0,0 +1,18 @@ + +def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { + def String cs = autobuilder.commonscripts() + def String difffolder = meta.folder() + def String jirakey = meta.productinfo.jirakey + if (isUnix()) { + sh """ + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + """ + } + else { + bat """ + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + """ + } +} + +return this; diff --git a/differences/json.groovy b/differences/json.groovy new file mode 100644 index 0000000..47d9cc5 --- /dev/null +++ b/differences/json.groovy @@ -0,0 +1,15 @@ +def comparefiles(importedjsonfile, currentjsonfile) { + env.IMPORTED_JSON="${importedjsonfile}" + env.CURRENT_JSON="${currentjsonfile}" + if (isUnix()) { + sh """ + # jq -S sorts json output + diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") + """ + } + else { + error "Windows not currently supporting json file comparison" + } +} + +return this; diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py new file mode 100644 index 0000000..0813356 --- /dev/null +++ b/differences/show_release_differences.py @@ -0,0 +1,226 @@ +''' +showreleasedifferences -- shortdesc + +showreleasedifferences generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import subprocess +import os +import sys +import re + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +gitExecutable = os.getenv("GIT_EXECUTABLE", default="git") + +class GitException(Exception): + def __init__(self, message): + self.message = message + + def __repr__(self): + return self.message + +def makeFolder(difffolder): + print("Creating folder (if not already there): {0}".format(difffolder)) + os.makedirs(difffolder, exist_ok = True) + +def printDivider(outfh, times = 1, length = 100): + for _ in range(times): + outfh.write("{0}\n".format("*" * length)) + +TABSIZE = 4 +def printTabbed(outfh, string, tabs = 0): + outfh.write("{0}{1}\n".format(" " * (TABSIZE * tabs), string)) + +def getGitLogDifferences(outfh, gitrootfolder, previousRelease, currentRelease, tripledot = False): + if tripledot: + releaseExpression = previousRelease + "..." + currentRelease + else: + releaseExpression = previousRelease + ".." + currentRelease + currentdir = os.getcwd() + os.chdir(gitrootfolder) + gitlog = [] + printTabbed(outfh, "Executing: {0} {1} {2} {3}".format(gitExecutable, "log", "--oneline", releaseExpression)) + proc = subprocess.Popen([gitExecutable, "log", "--oneline", releaseExpression], stdout=subprocess.PIPE) + while True: + line = proc.stdout.readline() + if line: + linestring = line.decode('utf-8').rstrip() + gitlog.append(linestring) + #printTabbed(outfh, linestring, 1) + else: + break + proc.wait() + if proc.returncode > 0: + raise GitException("Git executable returned an error") + os.chdir(currentdir) + return gitlog + +jiraprojectpattern = { + 'GIS' : ( re.compile("^.* +(?:[gG][iI][sS][ _-]*|[tT]rac \#*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *GIS-(\d+):* (.*)"$'), + ), + 'IMB': ( re.compile("^.* +(?:[Ii][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *IMB-(\d+):* (.*)"$'), + ), + 'HMBXR5601': ( re.compile("^.* +(?:[Hh][Mm][Bb][Xx][Rr]5601[ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBXR5601-(\d+):* (.*)"$'), + ), + 'SMB': ( re.compile("^.* +(?:[Ss][Mm][Bb][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *SMB-(\d+):* (.*)"$'), + ), + 'MCS': ( re.compile("^.* +(?:[Mm][Cc][Ss][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *MCS-(\d+):* (.*)"$'), + ), + 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), + ), + } + +mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") + +def processLog(gitlog, jirakeyprefix): + reverts = [] + merges = {} + other = [] + numbered = {} + (expectedpattern, revertpattern) = jiraprojectpattern[jirakeyprefix] + for line in gitlog: + expectedmatch = expectedpattern.match(line) + revertmatch = revertpattern.match(line) + mergematch = mergepattern.match(line) + if expectedmatch == None and revertmatch == None and mergematch == None: + other.append(line) + if revertmatch: + gisnumber = jirakeyprefix + "-" + revertmatch.group(2) + description = revertmatch.group(3) + reverts.append("Revert: {0} {1}".format(gisnumber, description)) + if mergematch: + merge = mergematch.group(2) + if not merge in merges: + merges[merge] = 1 + else: + merges[merge] += 1 + if expectedmatch: + gisnumber = jirakeyprefix + "-" + expectedmatch.group(1) + description = expectedmatch.group(2) + if not gisnumber in numbered: + numbered[gisnumber] = [] + numbered[gisnumber].append(description) + return (reverts, merges, other, numbered) + +def showDifferences(differences, jirakeyprefix, outfh): + (revertslog, mergeslog, otherlog, numberedlog) = differences + printDivider(outfh) + printTabbed(outfh, jirakeyprefix + "-Numbers Log:") + for (gisnumber, descriptions) in sorted(numberedlog.items()): + printTabbed(outfh, gisnumber, 1) + descriptions = numberedlog[gisnumber]; + for description in descriptions: + printTabbed(outfh, description, 2) + printDivider(outfh) + printTabbed(outfh, "Revert log:") + for reversion in sorted(revertslog): + printTabbed(outfh, reversion, 1) + printDivider(outfh) + printTabbed(outfh, "Merges log:") + for merge in sorted(mergeslog): + printTabbed(outfh, merge, 1) + printDivider(outfh) + printTabbed(outfh, "Other log:") + for other in sorted(otherlog): + printTabbed(outfh, other, 1) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-s", "--sourceversion", dest="sourceversion", action="store", default="origin/master", help="source version branch or tag to discover compare against [default: %(default)s]") + parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") + parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") + parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") + parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + gitrootfolder = args.gitrootfolder + sourceversion = args.sourceversion + previousversion = args.previousversion + jirakeyprefix = args.jirakeyprefix + outputoptions = args.outputoptions + difffolder = args.difffolder + + print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) + + if difffolder: + makeFolder(difffolder) + outfh = open(os.path.join(difffolder, "differences.txt"), 'w') + else: + outfh = sys.stdout + + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, previousversion, sourceversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, previousversion, sourceversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'opposite' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes in {2} that are not in {1} of the repository at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + if 'notcommon' in outputoptions.lower(): + printDivider(outfh, times = 4) + printTabbed(outfh, "Looking for changes that are not common to {1} or {2} at path: {0}".format(gitrootfolder, sourceversion, previousversion)) + gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) + showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) + + if difffolder: + outfh.close() + return 0 + except Exception as e: + print("Error while determining differences:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/file/utils.groovy b/file/utils.groovy new file mode 100644 index 0000000..0ffa47c --- /dev/null +++ b/file/utils.groovy @@ -0,0 +1,16 @@ + +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") + +// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents +def currentContents(file, newline = "${nl}", defaultContents = "") { + def contents = "" + if (fileExists("${file}")) { + contents = readFile(file: "${file}").trim() + newline + } + else { + contents = defaultContents + } + return contents +} + +return this; diff --git a/import/importer.groovy b/import/importer.groovy new file mode 100644 index 0000000..a9e28b2 --- /dev/null +++ b/import/importer.groovy @@ -0,0 +1,27 @@ + +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: specific(buildnumber.toString()), + target: destpath + ) + meta.importrelated(destpath) +} + +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { + copyArtifacts( + filter: filter, + fingerprintArtifacts: fingerprint, + projectName: jenkinsjob.trim(), + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: buildselector + ], + target: destpath + ) + meta.importrelated(destpath) +} + +return this; diff --git a/install/installhelper.groovy b/install/installhelper.groovy new file mode 100644 index 0000000..f523799 --- /dev/null +++ b/install/installhelper.groovy @@ -0,0 +1,42 @@ +// Given details about artifacts to import and details of installers to build, import artifacts that are required +def artifactsImporter(artifactDetails, installerDetails) { + artifactFetched = [:] + + for (installer in toBuild) { + def artifactKey = installerDetails.get(installer)[0] + //echo "To build: ${installer} use artifactKey: ${artifactKey}" + (sourceProject, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + env.importDest = destFolder + if (artifactFetched.get(artifactKey) == null) { + //echo "Artifact info: ${artifactInfo}" + copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" + def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" + def folderToCopy = "${destFolder}\\version" + echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" + fileOperations([folderCopyOperation(destinationFolderPath: componentVersionFolder, sourceFolderPath: folderToCopy)]) + //fileOperations([fileCopyOperation(excludes: '', flattenFiles: false, includes: filesToCopy, targetLocation: componentVersionFolder )]) + def gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + artifactVersion = gh.getSHAFromVersion("${destFolder}/version") + artifactFetched.put(artifactKey, artifactVersion) + } + } + return artifactFetched +} + +// Export any required binaries to the /bin folder +def exportBinaries(artifactDetails, installerDetails) { + for (installer in toBuild) { + (artifactSet, targetFolder, executionPath, ISMprojectFile, installerVariant, exportBinaries) = installerDetails.get(installer) + if (exportBinaries) { + (artifactsSourceProject, destinationFolder, artifactsToCopy, artifactsToNotCopy, binariesFolders) = artifactDetails.get(artifactSet) + for (binaryFolder in binariesFolders) { + binariesSourceFolder = "${destinationFolder}/${binaryFolder}" + binariesDestFolder = "${WORKSPACE}/bin/${artifactSet}/${binaryFolder}" + echo "Copying from: ${binariesSourceFolder} to: ${binariesDestFolder}" + fileOperations([folderCopyOperation(destinationFolderPath: binariesDestFolder, sourceFolderPath: binariesSourceFolder)]) + } + } + } +} + +return this; \ No newline at end of file diff --git a/meta/folder.groovy b/meta/folder.groovy new file mode 100644 index 0000000..8f26265 --- /dev/null +++ b/meta/folder.groovy @@ -0,0 +1,578 @@ +import java.nio.file.Paths +import java.nio.file.Path + +abstract class GISJsonSerialised implements Serializable { + def script + + GISJsonSerialised(script) { + this.script = script + } + + abstract LinkedHashMap serialisationdata() + + abstract void deserialiseJson(rawmeta) + + def saveJsonSerialised(String outputfile, int prettylevel = 4) { + def outJson = script.readJSON text: groovy.json.JsonOutput.toJson(serialisationdata()) + script.writeJSON(file: outputfile, json: outJson, pretty: prettylevel) + } +} + +class ProductInfo extends GISJsonSerialised { + private String productname + private String subcomponentname + String jirakey + + ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { + super(script) + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + + void set(String productname, String subcomponentname, String jirakey) { + this.productname = productname + this.subcomponentname = subcomponentname + this.jirakey = jirakey + } + + void deserialiseJson(rawmeta) { + productname = rawmeta['productname'] + subcomponentname = rawmeta['subcomponentname'] + jirakey = rawmeta['jirakey'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'productname' : productname, + 'subcomponentname' : subcomponentname, + 'jirakey' : jirakey, + ] + return data + } +} + +class GitInfo extends GISJsonSerialised { + private String sourcepath + private String sourcerevision + private String gitrevision + private String remoteurl + private String remoteinfo + + GitInfo(script, sourcepath = "", sourcerevision = "", gitrevision = "", remoteurl = "", remoteinfo = "") { + super(script) + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + void set(String sourcepath, String sourcerevision, String gitrevision, String remoteurl, String remoteinfo) { + this.sourcepath = sourcepath + this.sourcerevision = sourcerevision + this.gitrevision = gitrevision + this.remoteurl = remoteurl + this.remoteinfo = remoteinfo + } + + void deserialiseJson(rawmeta) { + sourcepath = rawmeta['sourcepath'] + sourcerevision = rawmeta['sourcerevision'] + gitrevision = rawmeta['gitrevision'] + remoteurl = rawmeta['remoteurl'] + remoteinfo = rawmeta['remoteinfo'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'sourcepath' : sourcepath, + 'sourcerevision' : sourcerevision, + 'gitrevision' : gitrevision, + 'remoteurl' : remoteurl, + 'remoteinfo' : remoteinfo, + ] + return data + } +} + +class JenkinsBuildInfo extends GISJsonSerialised { + private String buildjobname + private int buildnumber + private String buildmachine + private String builddate + + JenkinsBuildInfo(script, String buildjobname = "", int buildnumber = 0, String buildmachine = "", String builddate = "") { + super(script) + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + void set(String buildjobname, int buildnumber, String buildmachine, String builddate) { + this.buildjobname = buildjobname + this.buildnumber = buildnumber + this.buildmachine = buildmachine + this.builddate = builddate + } + + void deserialiseJson(rawmeta) { + buildjobname = rawmeta['buildjobname'] + buildnumber = rawmeta['buildnumber'] + buildmachine = rawmeta['buildmachine'] + builddate = rawmeta['builddate'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'buildjobname' : buildjobname, + 'buildnumber' : buildnumber, + 'buildmachine' : buildmachine, + 'builddate' : builddate, + ] + return data + } +} + +class BuildStampInfo extends GISJsonSerialised { + private String buildstampprefix + private int revision + String buildstamp + String buildstampcommasep + + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + super(script) + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + + void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + this.buildstampprefix = buildstampprefix + this.revision = revision + this.buildstamp = buildstamp + this.buildstampcommasep = buildstampcommasep + } + + void deserialiseJson(rawmeta) { + buildstampprefix = rawmeta['buildstampprefix'] + revision = rawmeta['revision'] + buildstamp = rawmeta['buildstamp'] + buildstampcommasep = rawmeta['buildstampcommasep'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'buildstampprefix' : buildstampprefix, + 'revision' : revision, + 'buildstamp' : buildstamp, + 'buildstampcommasep' : buildstampcommasep, + ] + return data + } +} + +class ReleaseInfo extends GISJsonSerialised { + private String releaserepoURL + private ArrayList releaseitems + + ReleaseInfo(script, releaserepoURL = "") { + super(script) + this.releaserepoURL = releaserepoURL + this.releaseitems = new ArrayList() + } + + void set(String releaserepoURL = "") { + this.releaserepoURL = releaserepoURL + } + + void addReleaseItem(String releaseitem) { + releaseitems.add(releaseitem) + } + + List releaseitemssserialised() { + def items = [] + for (ri in releaseitems) { + items.add(ri) + } + return items + } + + void deserialiseJson(rawmeta) { + releaserepoURL = rawmeta['releaserepoURL'] + releaseitems = new ArrayList() + for (rawreleaseitem in rawmeta['releaseitems']) { + releaseitems.add(rawreleaseitem) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'releaserepoURL' : releaserepoURL, + 'releaseitems' : releaseitemssserialised(), + ] + return data + } +} + +class ConfigurationInfo extends GISJsonSerialised { + private String configurationname + private String configurationURL + private String configurationversion + private String configurationSHA + + ConfigurationInfo(script, configurationname = "", configurationURL = "", configurationversion = "", configurationSHA = "") { + super(script) + this.configurationname = configurationname + this.configurationURL = configurationURL + this.configurationversion = configurationversion + this.configurationSHA = configurationSHA + } + + void deserialiseJson(rawmeta) { + configurationname = rawmeta['configurationname'] + configurationURL = rawmeta['configurationURL'] + configurationversion = rawmeta['configurationversion'] + configurationSHA = rawmeta['configurationSHA'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'configurationname' : configurationname, + 'configurationURL' : configurationURL, + 'configurationversion' : configurationversion, + 'configurationSHA' : configurationSHA, + ] + return data + } +} + +class ChecksumPair extends GISJsonSerialised { + private String filepath + private String sum + + ChecksumPair(script, String filepath = "", String sum = "") { + super(script) + this.filepath = filepath + this.sum = sum + } + + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + sum = rawmeta['sum'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + 'sum' : sum, + ] + return data + } +} + +class ChecksumSet extends GISJsonSerialised { + private String artifactpath + private ArrayList checksumpairs + + ChecksumSet(script, artifactpath = "") { + super(script) + this.artifactpath = artifactpath + checksumpairs = new ArrayList() + } + + ChecksumPair addChecksumPair(String filepath, String sum) { + def cp = new ChecksumPair(script, filepath, sum) + checksumpairs.add(cp) + return cp + } + + List checksumpairsserialised() { + def associated = [] + for (cp in checksumpairs) { + associated.add(cp.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + artifactpath = rawmeta['artifactpath'] + checksumpairs = new ArrayList() + for (rawchecksumpair in rawmeta['checksumpairs']) { + def csp = new ChecksumPair(script) + csp.deserialiseJson(rawchecksumpair) + checksumpairs.add(csp) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'artifactpath' : artifactpath, + 'checksumpairs' : checksumpairsserialised(), + ] + return data + } +} + +class ChecksumInfo extends GISJsonSerialised { + private ArrayList checksumsets + + ChecksumInfo(script) { + super(script) + checksumsets = new ArrayList() + } + + ChecksumSet addChecksumSet(String artifactpath) { + def cs = new ChecksumSet(script, artifactpath) + checksumsets.add(cs) + return cs + } + + List checksumsetsserialised() { + def associated = [] + for (cs in checksumsets) { + associated.add(cs.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + checksumsets = new ArrayList() + for (rawchecksumset in rawmeta['checksumsets']) { + def cs = new ChecksumSet(script) + cs.deserialiseJson(rawchecksumset) + checksumsets.add(cs) + } + } + + LinkedHashMap serialisationdata() { + def data = [ + 'checksumsets' : checksumsetsserialised(), + ] + return data + } +} + +class PublishableInfo extends GISJsonSerialised { + private String filepath + + PublishableInfo(script, String filepath = "") { + super(script) + this.filepath = filepath + } + + void deserialiseJson(rawmeta) { + filepath = rawmeta['filepath'] + } + + LinkedHashMap serialisationdata() { + def data = [ + 'filepath' : filepath, + ] + return data + } +} + +class MetaInfo extends GISJsonSerialised { + private String FORMAT = "0.1" + private String METAFOLDERNAME = "meta" + private String METAFILENAME = "info.json" + private String METAARTIFACTS = "${METAFOLDERNAME}/**" + + class RelatedMetaInfo extends GISJsonSerialised { + String offsetpath + MetaInfo meta + + RelatedMetaInfo(script, offsetpath = "", MetaInfo meta = null) { + super(script) + this.offsetpath = offsetpath + this.meta = meta + } + + void deserialiseJson(rawmeta) { + offsetpath = rawmeta['offsetpath'] + meta = new MetaInfo(script) + meta.deserialiseJson(rawmeta['meta']) + } + + LinkedHashMap serialisationdata() { + def data = [ + 'offsetpath' : offsetpath, + 'meta' : meta.serialisationdata(), + ] + return data + } + } + + private String format = FORMAT + ProductInfo productinfo + GitInfo gitinfo + JenkinsBuildInfo jenkinsbuildinfo + BuildStampInfo buildstampinfo + ReleaseInfo releaseinfo + ChecksumInfo checksuminfo + private ArrayList configurationinfo + private ArrayList publishableinfo + private ArrayList associatedmeta + + MetaInfo(script) { + super(script) + productinfo = new ProductInfo(script) + gitinfo = new GitInfo(script) + jenkinsbuildinfo = new JenkinsBuildInfo(script) + buildstampinfo = new BuildStampInfo(script) + releaseinfo = new ReleaseInfo(script) + checksuminfo = new ChecksumInfo(script) + configurationinfo = new ArrayList() + publishableinfo = new ArrayList() + associatedmeta = new ArrayList() + } + + List configurationinfoserialised() { + def configuration = [] + for (configurationitem in configurationinfo) { + configuration.add(configurationitem.serialisationdata()) + } + return configuration + } + + void addPublishable(String filepath) { + def PublishableInfo pi = new PublishableInfo(script, filepath) + publishableinfo.add(pi) + } + + void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { + def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + configurationinfo.add(ci) + } + + List publishableinfoserialised() { + def publishable = [] + for (publishableitem in publishableinfo) { + publishable.add(publishableitem.serialisationdata()) + } + return publishable + } + + List associatedmetaserialised() { + def associated = [] + for (metaitem in associatedmeta) { + associated.add(metaitem.serialisationdata()) + } + return associated + } + + void deserialiseJson(rawmeta) { + if (rawmeta['format'] != FORMAT) { + script.error("Unsupported meta format found at ${filepath}") + return + } + productinfo.deserialiseJson(rawmeta['productinfo']) + gitinfo.deserialiseJson(rawmeta['gitinfo']) + jenkinsbuildinfo.deserialiseJson(rawmeta['jenkinsbuildinfo']) + buildstampinfo.deserialiseJson(rawmeta['buildstampinfo']) + releaseinfo.deserialiseJson(rawmeta['releaseinfo']) + checksuminfo.deserialiseJson(rawmeta['checksuminfo']) + for (rawconfig in rawmeta['configuration']) { + def cf = new ConfigurationInfo(script) + cf.deserialiseJson(rawconfig) + configurationinfo.add(cf) + } + for (rawpublishable in rawmeta['publishableinfo']) { + def pb = new PublishableInfo(script) + pb.deserialiseJson(rawpublishable) + publishableinfo.add(pb) + } + for (rawrelatedmeta in rawmeta['associatedmeta']) { + def rm = new RelatedMetaInfo(script) + rm.deserialiseJson(rawrelatedmeta) + associatedmeta.add(rm) + } + } + + def deserialiseJson(String filepath) { + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + String relativize(String path) { + def Path thispath = Paths.get(path) + if (thispath.startsWith(script.WORKSPACE)) { + def Path wspath = Paths.get(script.WORKSPACE) + def Path wsrelativePath = wspath.relativize(thispath) + return wsrelativePath.toString() + } + return path + } + + String metafilepath() { + return METAFOLDERNAME + "/" + METAFILENAME + } + + MetaInfo importrelated(String destpath) { + MetaInfo mi = new MetaInfo(script) + + def String adjustedpath = relativize(destpath) + def importedmetafile = adjustedpath + "/" + metafilepath() + if (script.fileExists(importedmetafile)) { + mi.deserialiseJson(importedmetafile) + } + + RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) + associatedmeta.add(rmi) + + return mi + } + + LinkedHashMap serialisationdata() { + def data = [ + 'format' : format, + 'productinfo' : productinfo.serialisationdata(), + 'gitinfo' : gitinfo.serialisationdata(), + 'jenkinsbuildinfo' : jenkinsbuildinfo.serialisationdata(), + 'buildstampinfo' : buildstampinfo.serialisationdata(), + 'releaseinfo' : releaseinfo.serialisationdata(), + 'configuration' : configurationinfoserialised(), + 'checksuminfo' : checksuminfo.serialisationdata(), + 'publishableinfo' : publishableinfoserialised(), + 'associatedmeta' : associatedmetaserialised(), + ] + return data + } + + + String folder() { + return METAFOLDERNAME + } + + String generateartifacts(ArrayList releasedfromhere = new ArrayList(), + ArrayList releasedfromsubmeta = new ArrayList(), + String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" + ) { + def String artifactlist = METAARTIFACTS + releaseinfo.addReleaseItem(METAARTIFACTS) + for (releaseitem in releasedfromhere) { + artifactlist += "," + releaseitem + releaseinfo.addReleaseItem(releaseitem) + } + for (releaseitem in releasedfromsubmeta) { + artifactlist += "," + releaseitem + } + saveJsonSerialised(metafilepath) + return artifactlist + } +} + +MetaInfo metareset(script) { + echo "Resetting meta folder" + def mi = new MetaInfo(script) + + dir(mi.folder()) { + deleteDir() + } + return mi +} + +return this; diff --git a/platform/lineendings.groovy b/platform/lineendings.groovy new file mode 100644 index 0000000..9657b0d --- /dev/null +++ b/platform/lineendings.groovy @@ -0,0 +1,5 @@ + +// Put ${nl} into strings when wanting a newline +nl = System.properties['line.separator'] + +return this; diff --git a/publishing/utils.groovy b/publishing/utils.groovy new file mode 100644 index 0000000..145a7e9 --- /dev/null +++ b/publishing/utils.groovy @@ -0,0 +1,10 @@ + +// Generate a list of artifact files that are publishable to outside parties +def generatelist(meta, publishable) { + filelist = [] + for (filepath in publishable) { + meta.addPublishable(filepath) + } +} + +return this; diff --git a/release/registerrepo.groovy b/release/registerrepo.groovy new file mode 100644 index 0000000..ff4d85d --- /dev/null +++ b/release/registerrepo.groovy @@ -0,0 +1,12 @@ + +vc = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +// Prove the existence of the release repo - otherwise there is a large wait to cycle the build to correct the meta data +def registerrepo(meta, String releaserepo) { + if (releaserepo?.trim()) { + vc.gitCheckout(releaserepo, "origin/master", "tmp/releaserepo") + } + meta.releaseinfo.set(releaserepo) +} + +return this; diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy new file mode 100644 index 0000000..cb9354b --- /dev/null +++ b/versioncontrol/git.groovy @@ -0,0 +1,67 @@ +clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") + +String getGitInfo(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git rev-parse HEAD') + } +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { + reporootenv = env.REPO_ROOT + if (reporootenv != null && repoRoot != "") { + reporootpath = "${reporootenv.trim()}/${repoRoot}" + } else { + reporootpath = "" + echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" + } + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: revision] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], + [$class: 'CheckoutOption', timeout: 120], + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + ], + submoduleCfg: [], + ]) +} + +String getSHAFromVersion(String versionFolder) { + dir (versionFolder) { + srcRevision = readFile("gitrevision.txt").trim() + } + return srcRevision +} + +boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { + def keys = versions.keySet() + if (keys.size() < 1) { + return false + } + if (keys.size() == 1) { + return true + } + else { + version = "" + for (key in keys) { + nextversion = versions.get(key) + if (version == "") { + version = nextversion + } + if (nextversion != version) { + return false + } + } + } + return true +} + +return this diff --git a/versionnumbering/do.groovy b/versionnumbering/do.groovy new file mode 100644 index 0000000..9b4a42e --- /dev/null +++ b/versionnumbering/do.groovy @@ -0,0 +1,20 @@ + +def stamp(convertdetails) { + for (convertdetail in convertdetails) { + filetoconvert = convertdetail[0] + pattern = convertdetail[1] + replacement = convertdetail[2] + if (isUnix()) { + sh """ + "${PYTHON}" -u "${autobuilder.commonscripts()}/versionstamping/versionstamper.py" -f "${filetoconvert}" -p "${pattern}" -s "${replacement}" + """ + } + else { + bat """ + "%${PYTHON}%" -u "%${autobuilder.commonscripts()}%\\versionstamping\\versionstamper.py" -f "%${filetoconvert}%" -p "%${pattern}%" -s "%${replacement}%" + """ + } + } +} + +return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy new file mode 100644 index 0000000..59fe2a3 --- /dev/null +++ b/versionnumbering/generate.groovy @@ -0,0 +1,120 @@ + +def getGitRevision(String gitrootpath, int basebuildnumber) { + def int adjustedchangecount + def String gitrevision + def String remoteinfo + def String remoteurl + dir( gitrootpath) { + def int MAXVERSIONNUMBER = 65535 + + def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + echo "Change count: ${changecount}" + adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + echo "Adjusted change count: ${adjustedchangecount}" + + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + echo "Revision: ${gitrevision}" + + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + echo "Remote info: ${remoteinfo}" + + remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() + echo "Remote URL: ${remoteurl}" + } + return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] +} + +// Handle: +// origin/master +// origin/2.0.x +def getVersionPrefix(String sourcebranch) { + def int MAXVERSIONNUMBER = 65535 + + echo "Determining version number prefix from source branch name: ${sourcebranch}" + sourcebranch = sourcebranch.toLowerCase() + + def defaultversionprefix = "1.30000.0." + def splits = sourcebranch.split('/') + if (splits.size() < 1) { + echo "Source branch name is invalid. Defaulting to: ${defaultversionprefix}" + return defaultversionprefix + } + + def mastertypeversionprefix = "1.20000.0." + sourcebranch = splits.last() + if (sourcebranch.toLowerCase().find("master") != null) { + echo "Source branch name has 'master' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + + def splits2 = sourcebranch.split('\\.') + if (splits2.size() < 1) { + def thirdpartyversionprefix = "1.25000.0." + echo "Source branch name has no dots in it (eg. 1.9.2). Defaulting to: ${thirdpartyversionprefix}" + return thirdpartyversionprefix + } + + for (split in splits2) { + if (split == "x") { + echo "Source branch name has '.x' or 'x.' in it. Defaulting to: ${mastertypeversionprefix}" + return mastertypeversionprefix + } + } + + def integerstrings = sourcebranch.findAll(/\d+/) + def integers = [] + for (integerstring in integerstrings) { + int integer = integerstring.toInteger() + integer = integer % (MAXVERSIONNUMBER + 1) + integers.add(integer) + } + + def versionprefix = "" + if (integers.size() < 1) { + echo "This source branch name has no numbers in its name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else { + def int integernumbercount = 0 + for (integer in integers) { + versionprefix += integer + "." + integernumbercount += 1 + if (integernumbercount > 2) { + break + } + } + if (integernumbercount == 0) { + echo "No leading numbers were found in source branch name so defaulting to: ${defaultversionprefix}" + versionprefix = defaultversionprefix + } + else if (integernumbercount == 1) { + versionprefix += "0.0." + } + else if (integernumbercount == 2) { + versionprefix += "0." + } + } + echo "Determined version prefix to be: ${versionprefix}" + return versionprefix +} + +String generate(meta, String sourcerevision, String sourcepath, int basebuildnumber = 1000) { + def int gitrevisioncount + def String gitrevision + def String remoteinfo + def String remoteurl + + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(sourcepath, basebuildnumber) + + def versionprefix = getVersionPrefix(sourcerevision) + + def buildstamp = versionprefix + gitrevisioncount.toString() + echo "Build stamp is: ${buildstamp}" + + meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + + return buildstamp +} + +return this; \ No newline at end of file diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy new file mode 100644 index 0000000..44bfbde --- /dev/null +++ b/versionnumbering/metaversion.groovy @@ -0,0 +1,100 @@ + +def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") +def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") + +def prefixline(rootpath, filepath) { + def fileContents = readFile("${filepath}").trim() + def fileContentsNormalized = fileContents.normalize() + + def newFileContents = "" + def lines = fileContentsNormalized.readLines() + for (aline in lines) { + newFileContents += "${rootpath}" + "/" + aline + "${nl}" + } + return newFileContents +} + +def mergemeta(rootpath, versionFolder, outputFolder) { + echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" + + def differencesFile = "${versionFolder}/differences.txt" + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents + + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents + + def intrevision = 0 + if (fileExists("${versionFolder}/revision.txt")) { + def revision = readFile("${versionFolder}/revision.txt").trim() + intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + } + else { + intrevision = 1 + } + + return intrevision +} + +def appendcomponentinfo(rootpath, versionFolder, outputFolder) { + echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" + + def differencesFile = "${versionFolder}/differences.txt" + if (fileExists("${differencesFile}")) { + def newContents = readFile file: "${differencesFile}" + def metadifferencesFile = "${outputFolder}/differences.txt" + writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents + } + + def buildStampFile = "${versionFolder}/buildstamps.txt" + def buildStampFileContents = readFile("${buildStampFile}").trim() + + def remoteurlFile = "${versionFolder}/remoteurl.txt" + def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() + + def gitRevisionFile = "${versionFolder}/gitrevision.txt" + def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() + + def releaseJobNameFile = "${versionFolder}/releasejobname.txt" + def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() + + def metabuildStampFile = "${outputFolder}/buildstamps.txt" + writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents + def revision = readFile("${versionFolder}/revision.txt").trim() + def intrevision = revision.toInteger() + echo "Revision read in: ${intrevision}" + + return intrevision +} + +// Each version folder should have been created using version creation scripts in this Common_scripts folder +// as their are well-known files in each folder +def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { + def totalrevision = 0 + dir("${outputFolder}") { + deleteDir() + } + for (artifactItem in artifactList) { + (rootpath, versionFolder) = artifactItem + echo "Inspecting folder: ${versionFolder}" + def metaversionsignaturefile = "${versionFolder}/metaversion.txt" + // If the sub project is a meta job + def intrevision = 0 + if (fileExists("${metaversionsignaturefile}")) { + intrevision = mergemeta(rootpath, versionFolder, outputFolder) + } + else { + intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) + } + totalrevision += intrevision + } + echo "Accumulated revision is: ${totalrevision}" + writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() + writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) +} + +return this; \ No newline at end of file diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py new file mode 100644 index 0000000..3d2a80d --- /dev/null +++ b/versionnumbering/versioninfo.py @@ -0,0 +1,180 @@ +# encoding: utf-8 +''' +versionnumbering -- shortdesc + +versionnumbering generates a version folder with a common set of version information files in it + +@author: dan + +@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. +''' +import os +import sys +import shutil +import subprocess +import re +import datetime + +from argparse import ArgumentParser +from argparse import RawDescriptionHelpFormatter + +__all__ = [] +__version__ = 0.1 +__date__ = '2017-11-29' +__updated__ = '2017-11-29' + +# Handle: +# origin/master +# origin/2.0.x +def getVersionPrefix(sourcebranch): + print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) + + defaultversionprefix = "1.30000.0." + mastertypeversionprefix = "1.20000.0." + thirdpartyversionprefix = "1.25000.0." + splits = sourcebranch.split('/') + if len(splits) < 1: + print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) + return defaultversionprefix + + sourcebranch = splits[-1] + + if sourcebranch.lower().find("master") != -1: # Special case + return mastertypeversionprefix + + splits2 = re.split(r'(\d+)', sourcebranch) + splits = [] + hasdot = False + for split in splits2: + if split.isdigit(): + splits.append(split) + if split == ".": + hasdot = True + + if not hasdot: + print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) + return thirdpartyversionprefix + + versionprefix = "" + if len(splits) < 1: + print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + splitnumbercount = 0 + isIntermediateReleaseBranch = False + for split in splits: + if split.isdigit(): + versionprefix += split + "." + splitnumbercount += 1 + else: + if split == "x": + isIntermediateReleaseBranch = True + break + if splitnumbercount > 2: + break + if splitnumbercount == 0: + print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) + versionprefix = defaultversionprefix + else: + if isIntermediateReleaseBranch: + versionprefix = mastertypeversionprefix + elif splitnumbercount == 1: + versionprefix += "0.0." + elif splitnumbercount == 2: + versionprefix += "0." + + print("Determined version prefix to be: {0}".format(versionprefix)) + return versionprefix + +def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): + with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: + f.write(versionprefix) + + with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: + f.write(str(gitrevisioncount)) + + with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: + f.write(gitrevision) + + with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: + f.write(remoteinfo) + + with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: + f.write(remoteurl) + + with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: + f.write(buildstamp) + + buildstampcommasep = buildstamp.replace(".", ",") + with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: + f.write(buildstampcommasep) + + with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: + f.write(str(datetime.datetime.now())) + +def main(argv=None): # IGNORE:C0111 + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + program_name = os.path.basename(sys.argv[0]) + program_version = "v%s" % __version__ + program_build_date = str(__updated__) + program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) + program_shortdesc = __import__('__main__').__doc__.split("\n")[1] + program_license = '''%s + + Created by dan on %s. + Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. + + Distributed on an "AS IS" basis without warranties + or conditions of any kind, either express or implied. + +USAGE +''' % (program_shortdesc, str(__date__)) + + try: + # Setup argument parser + parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) + parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") + parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") + parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") + parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") + + # Process arguments + args = parser.parse_args() + + try: + sourcebranch = args.sourcebranch + versionfolder = args.versionfolder + gitrootfolder = args.gitrootfolder + baseversionnumber = args.baseversionnumber + + print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) + print("Base version number:", baseversionnumber) + + (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) + + versionprefix = getVersionPrefix(sourcebranch) + + buildstamp = versionprefix + str(gitrevisioncount) + print("Build stamp is: {0}".format(buildstamp)) + + writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) + + return 0 + except Exception as e: + print("Error while creating version info:", e.__repr__()) + return 1 + + except Exception as e: + indent = len(program_name) * " " + print(program_name + ": " + repr(e) + "\n") + print(indent + " for help use --help") + return 2 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/versionstamping/.project b/versionstamping/.project new file mode 100644 index 0000000..74928d6 --- /dev/null +++ b/versionstamping/.project @@ -0,0 +1,17 @@ + + + versionstamping + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/versionstamping/.pydevproject b/versionstamping/.pydevproject new file mode 100644 index 0000000..746d0f0 --- /dev/null +++ b/versionstamping/.pydevproject @@ -0,0 +1,8 @@ + + + +/${PROJECT_DIR_NAME} + +python 3.0 +python35 + diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py new file mode 100644 index 0000000..3e59c6f --- /dev/null +++ b/versionstamping/versionstamper.py @@ -0,0 +1,41 @@ +import argparse +import sys +import shutil +from builtins import FileNotFoundError + +def main(argv): + parser = argparse.ArgumentParser(description="Look for version number pattern in file and replace with autobuild version number") + + # Call once arguments to allow or disallow something + parser.add_argument('-f', '--filetoconvert', help='The file to convert') + parser.add_argument('-p', '--pattern', default='1.30000.0.0', help='The pattern to search for and replace') + parser.add_argument('-s', '--substitute', help='The new version number that will substitute the pattern') + + args = parser.parse_args(argv) + + filetoconvert = args.filetoconvert + pattern = args.pattern + substitute = args.substitute + + print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) + + filecopy = filetoconvert + ".orig" + try: + shutil.move(filetoconvert, filecopy) + except FileNotFoundError: + print("Could not find file: {0}\n".format(filetoconvert)) + return 1 + + with open(filetoconvert, 'x') as f: + with open(filecopy,'r') as c: + fileread = c.read() + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + #print ("DEBUG: %s "%converted) + print("Confirmed %s instances of %s post-conversion"%(converted.count(substitute),substitute)) + f.write(converted) + + return 0 + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 9dc60d37cb475bfcadbe920d927256aef8dd8764 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 19:10:54 +0000 Subject: [PATCH 306/558] Check out release repos --- autobuilder.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 2a04925..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -25,17 +27,21 @@ def getautobuildpath() { return autobuildroot } +def getmetafolderscript() { + return metafolderscript +} + // Start up boilderplate def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - def meta = metafolder.metareset(script) + def meta = getmetafolderscript().metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) From 63b97ce0d6b9d3dee10c4eca82e5c43373fb87c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:19:18 +0000 Subject: [PATCH 307/558] Check out release repos --- autobuilder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/autobuilder.groovy b/autobuilder.groovy index 9f0aece..885d243 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,6 +28,7 @@ def getautobuildpath() { } def getmetafolderscript() { + echo "metafolderscript is type: ${metafolderscript.toClass()}" return metafolderscript } From 67cb05649fec253dd76362fcd8940f9a68daf97e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:23:17 +0000 Subject: [PATCH 308/558] Check out release repos --- autobuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index 885d243..e40a67e 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,7 +28,7 @@ def getautobuildpath() { } def getmetafolderscript() { - echo "metafolderscript is type: ${metafolderscript.toClass()}" + echo "metafolderscript is type: ${metafolderscript.getClass()}" return metafolderscript } From 65094bf04d5be86c17241373a07e1e1d57244781 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 21:41:01 +0000 Subject: [PATCH 309/558] Check out release repos --- meta/folder.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index de445f1..458156d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -601,6 +601,7 @@ MetaInfo metareset(script) { MetaInfo metafromfolder(String folder = "") { MetaInfo buildmeta = new MetaInfo(this) buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta } return this; From 8e7fcd689dde683c93d2db4932805d23dfaa9a2c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Nov 2018 22:35:37 +0000 Subject: [PATCH 310/558] Remove debug --- autobuilder.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/autobuilder.groovy b/autobuilder.groovy index e40a67e..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -28,7 +28,6 @@ def getautobuildpath() { } def getmetafolderscript() { - echo "metafolderscript is type: ${metafolderscript.getClass()}" return metafolderscript } From b81ad461044787a2d20285ba3ee2ddaeb1bec65f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 09:07:06 +0000 Subject: [PATCH 311/558] Create a version number for the all build --- artifacts/metaexport.groovy | 8 ++++++-- meta/folder.groovy | 10 +++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b6a832b..e87e3d9 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,6 +4,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() + int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -13,11 +14,14 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + revisiontotal += importedmeta.buildstampinfo.revision artifactFolders.add("${destFolder}/**") } } - return artifactFolders; + def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + revisiontotal = revisiontotal % (versiongenerator.MAXVERSIONNUMBER + 1) + return [artifactFolders, revisiontotal] } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 458156d..74ac7c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -137,7 +137,7 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix - private int revision + int revision String buildstamp String buildstampcommasep @@ -156,6 +156,14 @@ class BuildStampInfo extends GISJsonSerialised { this.buildstampcommasep = buildstampcommasep } + void changerevision(int newrevision) { + revision = newrevision + buildstampsplit = buildstamp.split('.') + buildstampsplit[3] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + buildstampcommasep = buildstampsplit.join(',') + } + void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] From 1955cdd11041732230f0cfb827a63467681b1a52 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 11:37:10 +0000 Subject: [PATCH 312/558] Correct scope of MAXVERSIONNUMBER --- versionnumbering/generate.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 59fe2a3..cf21fd0 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,3 +1,4 @@ +int MAXVERSIONNUMBER = 65535 def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount @@ -5,8 +6,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) @@ -28,8 +27,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { - def int MAXVERSIONNUMBER = 65535 - echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() From b06f0855cf66c1c6260ad59eefc035d093d88f4a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 13:10:26 +0000 Subject: [PATCH 313/558] Correct scope of MAXVERSIONNUMBER --- artifacts/metaexport.groovy | 2 +- versionnumbering/generate.groovy | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index e87e3d9..b4d4a6d 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -20,7 +20,7 @@ def combineSubjobArtifacts(meta, subJobs) { } } def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.MAXVERSIONNUMBER + 1) + revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) return [artifactFolders, revisiontotal] } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index cf21fd0..9df06d8 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,4 +1,8 @@ -int MAXVERSIONNUMBER = 65535 + +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount @@ -8,7 +12,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -62,7 +66,7 @@ def getVersionPrefix(String sourcebranch) { def integers = [] for (integerstring in integerstrings) { int integer = integerstring.toInteger() - integer = integer % (MAXVERSIONNUMBER + 1) + integer = integer % (maxversionnumber() + 1) integers.add(integer) } From d479c10bf4b769ef53dca4903e17b52ad7c048af Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:11:08 +0000 Subject: [PATCH 314/558] Load in artifacts for release --- meta/folder.groovy | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 74ac7c4..822472d 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -594,6 +594,17 @@ class MetaInfo extends GISJsonSerialised { releasemetacollection = releasewalkerp(releasemetacollection, startpath) return releasemetacollection } + + Map releasewalkerunique(String startpath = "") { + Map releasemetauniquecollection = new Map() + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(releasemetacollection, startpath) + for (relatedmetainfo in releasemetacollection) { + String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetauniquecollection + } } MetaInfo metareset(script) { From 3027bb85e64c0ee1da71be7349dd18be0010ec63 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:17:09 +0000 Subject: [PATCH 315/558] Load in artifacts for release --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 822472d..8f8ea0a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -596,7 +596,7 @@ class MetaInfo extends GISJsonSerialised { } Map releasewalkerunique(String startpath = "") { - Map releasemetauniquecollection = new Map() + Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { From 55c4ce8c81b3263c953dd019a0ca72f9abc82cc0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 17:40:47 +0000 Subject: [PATCH 316/558] Load in artifacts for release --- meta/folder.groovy | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f8ea0a..24bcbf5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -573,32 +573,34 @@ class MetaInfo extends GISJsonSerialised { } // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java - private ArrayList releasewalkerp(ArrayList releasemetacollection, String pathsofar) { - if (releaseinfo.releaserepoURL) { - RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) - releasemetacollection.add(rmi) - } - if (pathsofar) { - pathsofar += "/" - } - for (relatedmetainfo in associatedmeta) { - MetaInfo othermeta = relatedmetainfo.meta - String newpathsofar = pathsofar + relatedmetainfo.offsetpath - releasemetacollection = othermeta.releasewalkerp(releasemetacollection, newpathsofar) + private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { + if (maxdepth > 0) { + if (releaseinfo.releaserepoURL) { + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + releasemetacollection.add(rmi) + } + if (pathsofar) { + pathsofar += "/" + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + relatedmetainfo.offsetpath + releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) + } } return releasemetacollection } - ArrayList releasewalker(String startpath = "") { + ArrayList releasewalker(int maxdepth, String startpath = "") { ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(releasemetacollection, startpath) + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) return releasemetacollection } - Map releasewalkerunique(String startpath = "") { + Map releasewalkerunique(int maxdepth, String startpath = "") { Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() - releasemetacollection = releasewalkerp(releasemetacollection, startpath) + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname releasemetauniquecollection.put(uniquestring, relatedmetainfo) From 7982ce90720ffbc7f307e08cb3c92cea3a90bf81 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 19:24:10 +0000 Subject: [PATCH 317/558] Perform git push operations --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 24bcbf5..b8120a6 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -158,7 +158,7 @@ class BuildStampInfo extends GISJsonSerialised { void changerevision(int newrevision) { revision = newrevision - buildstampsplit = buildstamp.split('.') + def List buildstampsplit = buildstamp.split('.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') From 060234eb5f8ffcf45e6f6d055de6799ea7de177b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Nov 2018 19:44:11 +0000 Subject: [PATCH 318/558] Perform git push operations --- meta/folder.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index b8120a6..cfa0a6e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -157,11 +157,13 @@ class BuildStampInfo extends GISJsonSerialised { } void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") revision = newrevision def List buildstampsplit = buildstamp.split('.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') + script.echo("Build stamp now ${buildstamp}") } void deserialiseJson(rawmeta) { From 333f65998462d94893f5cd9d76a53d1c67dc3094 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 00:33:39 +0000 Subject: [PATCH 319/558] Calculate version number earlier --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cfa0a6e..ed17a91 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -159,7 +159,7 @@ class BuildStampInfo extends GISJsonSerialised { void changerevision(int newrevision) { script.echo("Build stamp currently ${buildstamp}") revision = newrevision - def List buildstampsplit = buildstamp.split('.') + def List buildstampsplit = buildstamp.split('\\.') buildstampsplit[3] = newrevision.toString() buildstamp = buildstampsplit.join('.') buildstampcommasep = buildstampsplit.join(',') From 002ab542e035a122ce43cd06384ed54bc11c5512 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:06:46 +0000 Subject: [PATCH 320/558] Remote URL fetch was not working --- meta/folder.groovy | 2 +- versionnumbering/generate.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ed17a91..2640567 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,4 +1,4 @@ -import java.nio.file.Paths +urimport java.nio.file.Paths import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 9df06d8..f185c40 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -21,7 +21,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From 7b847a84ab77eede0212af622496237d564ba0e5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:10:04 +0000 Subject: [PATCH 321/558] Remote URL fetch was not working --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2640567..ed17a91 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -1,4 +1,4 @@ -urimport java.nio.file.Paths +import java.nio.file.Paths import java.nio.file.Path abstract class GISJsonSerialised implements Serializable { From 78ad47a964afcca9e7016c14876a71373d574d41 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 08:36:20 +0000 Subject: [PATCH 322/558] Remove comma separated buildstamp store --- meta/folder.groovy | 17 ++++++++--------- versionnumbering/generate.groovy | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ed17a91..76fbe43 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -139,30 +139,31 @@ class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix int revision String buildstamp - String buildstampcommasep - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } - void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } + String buildstampcommasep() { + buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + void changerevision(int newrevision) { script.echo("Build stamp currently ${buildstamp}") revision = newrevision def List buildstampsplit = buildstamp.split('\\.') - buildstampsplit[3] = newrevision.toString() + buildstampsplit[-1] = newrevision.toString() buildstamp = buildstampsplit.join('.') - buildstampcommasep = buildstampsplit.join(',') script.echo("Build stamp now ${buildstamp}") } @@ -170,7 +171,6 @@ class BuildStampInfo extends GISJsonSerialised { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] - buildstampcommasep = rawmeta['buildstampcommasep'] } LinkedHashMap serialisationdata() { @@ -178,7 +178,6 @@ class BuildStampInfo extends GISJsonSerialised { 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, - 'buildstampcommasep' : buildstampcommasep, ] return data } diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f185c40..a7b583a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -113,7 +113,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum echo "Build stamp is: ${buildstamp}" meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) return buildstamp } From 7bf878097c5a520a8a1da85b684b04953a005a24 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 12:48:41 +0000 Subject: [PATCH 323/558] Correct buildstamp comma sep generation --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 76fbe43..570a3ea 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -154,7 +154,7 @@ class BuildStampInfo extends GISJsonSerialised { } String buildstampcommasep() { - buildstampcommasep = buildstamp.replace(".", ",") + def String buildstampcommasep = buildstamp.replace(".", ",") return buildstampcommasep } From 583f38f347d4a336295dfad3a7d0b2147be256d0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 13:23:40 +0000 Subject: [PATCH 324/558] Remove redundant script; Correct artifact export of release job --- versionnumbering/metaversion.groovy | 100 ---------------------------- 1 file changed, 100 deletions(-) delete mode 100644 versionnumbering/metaversion.groovy diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy deleted file mode 100644 index 44bfbde..0000000 --- a/versionnumbering/metaversion.groovy +++ /dev/null @@ -1,100 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - -def prefixline(rootpath, filepath) { - def fileContents = readFile("${filepath}").trim() - def fileContentsNormalized = fileContents.normalize() - - def newFileContents = "" - def lines = fileContentsNormalized.readLines() - for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + "${nl}" - } - return newFileContents -} - -def mergemeta(rootpath, versionFolder, outputFolder) { - echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents - - def intrevision = 0 - if (fileExists("${versionFolder}/revision.txt")) { - def revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - } - else { - intrevision = 1 - } - - return intrevision -} - -def appendcomponentinfo(rootpath, versionFolder, outputFolder) { - echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents - } - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = readFile("${buildStampFile}").trim() - - def remoteurlFile = "${versionFolder}/remoteurl.txt" - def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - - def gitRevisionFile = "${versionFolder}/gitrevision.txt" - def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - def releaseJobNameFile = "${versionFolder}/releasejobname.txt" - def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - def revision = readFile("${versionFolder}/revision.txt").trim() - def intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - - return intrevision -} - -// Each version folder should have been created using version creation scripts in this Common_scripts folder -// as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - def totalrevision = 0 - dir("${outputFolder}") { - deleteDir() - } - for (artifactItem in artifactList) { - (rootpath, versionFolder) = artifactItem - echo "Inspecting folder: ${versionFolder}" - def metaversionsignaturefile = "${versionFolder}/metaversion.txt" - // If the sub project is a meta job - def intrevision = 0 - if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta(rootpath, versionFolder, outputFolder) - } - else { - intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) - } - totalrevision += intrevision - } - echo "Accumulated revision is: ${totalrevision}" - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) -} - -return this; \ No newline at end of file From 56ab82876073c7bf98f3942b88261660c4ebf8e7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:03:12 +0000 Subject: [PATCH 325/558] Alter differences file generation; Merge differences when importing --- differences/do.groovy | 6 +++--- differences/show_release_differences.py | 11 +++++------ meta/folder.groovy | 22 +++++++++++++++++++--- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/differences/do.groovy b/differences/do.groovy index 35e0980..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,16 +1,16 @@ def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { def String cs = autobuilder.commonscripts() - def String difffolder = meta.folder() + def String difffile = meta.difffilepath() def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0813356..fee6f7b 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -172,7 +172,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") + parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +183,13 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - difffolder = args.difffolder + difffile = args.difffile print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - if difffolder: - makeFolder(difffolder) - outfh = open(os.path.join(difffolder, "differences.txt"), 'w') + if difffile: + outfh = open(difffile), 'w') else: outfh = sys.stdout diff --git a/meta/folder.groovy b/meta/folder.groovy index 570a3ea..8bd80d3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,8 +19,8 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - private String productname - private String subcomponentname + String productname + String subcomponentname String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { @@ -388,6 +388,7 @@ class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" private static String METAFILENAME = "info.json" + private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { @@ -519,7 +520,11 @@ class MetaInfo extends GISJsonSerialised { static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } - + + static String difffilepath() { + return METAFOLDERNAME + "/" + METADIFFFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) @@ -532,6 +537,17 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) + def importedmetadifffile = adjustedpath + "/" + difffilepath() + if (script.fileExists(importedmetadifffile)) { + def String newcontents = script.readFile(file: importedmetadifffile) + def String currentcontents = "" + if (script.fileExists(difffilepath())) { + currentcontents = script.readFile(file: difffilepath()) + } + + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " mi.productinfo.componentname + ":\n" + newcontents) + } + return mi } From 56fab48730ee1813f2790f2857bc7130ec430f96 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:05:23 +0000 Subject: [PATCH 326/558] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 8bd80d3..2269a77 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " mi.productinfo.componentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.componentname + ":\n" + newcontents) } return mi From e079fb83b747468434bc0b5f4e65d6f2780f0bf0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:31:21 +0000 Subject: [PATCH 327/558] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2269a77..ac2ab9a 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.componentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) } return mi From 6f1bbf0aef834682debfce53be0e317311c30bfe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:36:42 +0000 Subject: [PATCH 328/558] Alter differences file generation; Merge differences when importing --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ac2ab9a..9c5367e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -545,7 +545,7 @@ class MetaInfo extends GISJsonSerialised { currentcontents = script.readFile(file: difffilepath()) } - script.writeFile(file: difffilepath(), currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) + script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) } return mi From 76bcd9fdea770e94dac367703efece5a1f8db87c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:41:49 +0000 Subject: [PATCH 329/558] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index fee6f7b..0600753 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -189,7 +189,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: - outfh = open(difffile), 'w') + outfh = open(difffile, 'w') else: outfh = sys.stdout From eb295970556ebef75e7e0a0112de745a26ac6ea2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 14:51:43 +0000 Subject: [PATCH 330/558] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0600753..ddf8c0d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,10 +29,6 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(difffolder): - print("Creating folder (if not already there): {0}".format(difffolder)) - os.makedirs(difffolder, exist_ok = True) - def printDivider(outfh, times = 1, length = 100): for _ in range(times): outfh.write("{0}\n".format("*" * length)) @@ -208,7 +204,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if difffolder: + if difffile: outfh.close() return 0 except Exception as e: From a68a9027717abb013386d18d5774ae25c89454b0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 15:05:23 +0000 Subject: [PATCH 331/558] Add HMBFDGMA jira comment regular expression --- differences/show_release_differences.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index ddf8c0d..80b636e 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -80,6 +80,9 @@ jiraprojectpattern = { 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), + 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") From 2f5617caea148498daadd3f4faf6aca95727777f Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Thu, 29 Nov 2018 15:13:15 +0000 Subject: [PATCH 332/558] Merge commit '5bb08476af818e51c87b40798049bf249bc2b6a4' into 1.10003.j20-comscr # Conflicts: # autobuild/Common_scripts/versionnumbering/metaversion.groovy --- artifacts/metaexport.groovy | 8 +- autobuilder.groovy | 10 ++- differences/do.groovy | 6 +- differences/show_release_differences.py | 20 +++-- import/importer.groovy | 4 +- meta/folder.groovy | 103 +++++++++++++++++++----- versionnumbering/generate.groovy | 17 ++-- versionnumbering/metaversion.groovy | 100 ----------------------- 8 files changed, 122 insertions(+), 146 deletions(-) delete mode 100644 versionnumbering/metaversion.groovy diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b6a832b..b4d4a6d 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,6 +4,7 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() + int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -13,11 +14,14 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + revisiontotal += importedmeta.buildstampinfo.revision artifactFolders.add("${destFolder}/**") } } - return artifactFolders; + def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) + return [artifactFolders, revisiontotal] } return this; diff --git a/autobuilder.groovy b/autobuilder.groovy index 2a04925..9f0aece 100644 --- a/autobuilder.groovy +++ b/autobuilder.groovy @@ -4,6 +4,8 @@ autobuildroot = "${WORKSPACE}/autobuild" commonscriptsroot = "${autobuildroot}/Common_scripts" +metafolderscript = null + def conditionalclean(doclean = "${CLEAN_WORKSPACE}" == "true") { if (doclean) { echo "Cleaning workspace" @@ -25,17 +27,21 @@ def getautobuildpath() { return autobuildroot } +def getmetafolderscript() { + return metafolderscript +} + // Start up boilderplate def start(script, productname = PRODUCT_NAME, subcomponentname = SUBCOMPONENT_NAME, jirakey = JIRA_KEY, releaserepo = RELEASE_REPO, autobuildpath = autobuildroot, commonscriptspath = commonscriptsroot) { echo "Starting...." autobuildroot = autobuildpath commonscriptsroot = commonscriptspath + metafolderscript = load("${commonscripts()}/meta/folder.groovy") conditionalclean() - def metafolder = load("${commonscripts()}/meta/folder.groovy") - def meta = metafolder.metareset(script) + def meta = getmetafolderscript().metareset(script) meta.productinfo.set(productname, subcomponentname, jirakey) def now = new Date() now.format("yyMMdd.HHmm", TimeZone.getTimeZone('UTC')) diff --git a/differences/do.groovy b/differences/do.groovy index 35e0980..32f18b5 100644 --- a/differences/do.groovy +++ b/differences/do.groovy @@ -1,16 +1,16 @@ def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { def String cs = autobuilder.commonscripts() - def String difffolder = meta.folder() + def String difffile = meta.difffilepath() def String jirakey = meta.productinfo.jirakey if (isUnix()) { sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } else { bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -d "${difffolder}" + "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" """ } } diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 0813356..80b636e 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -29,10 +29,6 @@ class GitException(Exception): def __repr__(self): return self.message -def makeFolder(difffolder): - print("Creating folder (if not already there): {0}".format(difffolder)) - os.makedirs(difffolder, exist_ok = True) - def printDivider(outfh, times = 1, length = 100): for _ in range(times): outfh.write("{0}\n".format("*" * length)) @@ -84,6 +80,9 @@ jiraprojectpattern = { 'HMBFDHV': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Hh][Vv][ _-]*|\#\s*)(\d+):*(.*)$"), re.compile('^(\S*) Revert " *HMBFDHV-(\d+):* (.*)"$'), ), + 'HMBFDGMA': ( re.compile("^.* +(?:[Hh][Mm][Bb][Ff][Dd][Gg][Mm][Aa][ _-]*|\#\s*)(\d+):*(.*)$"), + re.compile('^(\S*) Revert " *HMBFDGMA-(\d+):* (.*)"$'), + ), } mergepattern = re.compile("^(.*)((?:Merge branch '.*'|Merge remote-tracking branch '.*'|Merge commit '.*'|Add '.*' from commit).*)$") @@ -172,7 +171,7 @@ USAGE parser.add_argument("-p", "--previousversion", dest="previousversion", action="store", default="origin/master", help="source branch or tag to compare with [default: %(default)s]") parser.add_argument("-j", "--jirakeyprefix", dest="jirakeyprefix", action="store", default="GIS", help="jira key prefix [default: %(default)s]") parser.add_argument("-o", "--outputoptions", dest="outputoptions", action="store", default="", help="output options: Can be: opposite notcommon [default: %(default)s]") - parser.add_argument("-d", "--difffolder", dest="difffolder", action="store", default="", help="folder where differences file stored [default: %(default)s]") + parser.add_argument("-f", "--difffile", dest="difffile", action="store", default="", help="file where differences file stored [default: %(default)s]") # Process arguments args = parser.parse_args() @@ -183,14 +182,13 @@ USAGE previousversion = args.previousversion jirakeyprefix = args.jirakeyprefix outputoptions = args.outputoptions - difffolder = args.difffolder + difffile = args.difffile print("Git root:", gitrootfolder, "Source version:", sourceversion, "Previous source version:", previousversion) - print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Version folder:", difffolder) + print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) - if difffolder: - makeFolder(difffolder) - outfh = open(os.path.join(difffolder, "differences.txt"), 'w') + if difffile: + outfh = open(difffile, 'w') else: outfh = sys.stdout @@ -209,7 +207,7 @@ USAGE gitlog = getGitLogDifferences(outfh, gitrootfolder, sourceversion, previousversion, tripledot = True) showDifferences(processLog(gitlog, jirakeyprefix), jirakeyprefix, outfh) - if difffolder: + if difffile: outfh.close() return 0 except Exception as e: diff --git a/import/importer.groovy b/import/importer.groovy index a9e28b2..4d24d30 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -7,7 +7,7 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +21,7 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - meta.importrelated(destpath) + return meta.importrelated(destpath) } return this; diff --git a/meta/folder.groovy b/meta/folder.groovy index 8f26265..9c5367e 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,8 +19,8 @@ abstract class GISJsonSerialised implements Serializable { } class ProductInfo extends GISJsonSerialised { - private String productname - private String subcomponentname + String productname + String subcomponentname String jirakey ProductInfo(script, productname = "", subcomponentname = "", jirakey = "") { @@ -137,30 +137,40 @@ class JenkinsBuildInfo extends GISJsonSerialised { class BuildStampInfo extends GISJsonSerialised { private String buildstampprefix - private int revision + int revision String buildstamp - String buildstampcommasep - BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "", buildstampcommasep = "") { + BuildStampInfo(script, buildstampprefix = "", revision = 0, buildstamp = "") { super(script) this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep } - void set(String buildstampprefix, int revision, String buildstamp, String buildstampcommasep) { + void set(String buildstampprefix, int revision, String buildstamp) { this.buildstampprefix = buildstampprefix this.revision = revision this.buildstamp = buildstamp - this.buildstampcommasep = buildstampcommasep + } + + String buildstampcommasep() { + def String buildstampcommasep = buildstamp.replace(".", ",") + return buildstampcommasep + } + + void changerevision(int newrevision) { + script.echo("Build stamp currently ${buildstamp}") + revision = newrevision + def List buildstampsplit = buildstamp.split('\\.') + buildstampsplit[-1] = newrevision.toString() + buildstamp = buildstampsplit.join('.') + script.echo("Build stamp now ${buildstamp}") } void deserialiseJson(rawmeta) { buildstampprefix = rawmeta['buildstampprefix'] revision = rawmeta['revision'] buildstamp = rawmeta['buildstamp'] - buildstampcommasep = rawmeta['buildstampcommasep'] } LinkedHashMap serialisationdata() { @@ -168,15 +178,14 @@ class BuildStampInfo extends GISJsonSerialised { 'buildstampprefix' : buildstampprefix, 'revision' : revision, 'buildstamp' : buildstamp, - 'buildstampcommasep' : buildstampcommasep, ] return data } } class ReleaseInfo extends GISJsonSerialised { - private String releaserepoURL - private ArrayList releaseitems + String releaserepoURL + ArrayList releaseitems ReleaseInfo(script, releaserepoURL = "") { super(script) @@ -376,10 +385,11 @@ class PublishableInfo extends GISJsonSerialised { } class MetaInfo extends GISJsonSerialised { - private String FORMAT = "0.1" - private String METAFOLDERNAME = "meta" - private String METAFILENAME = "info.json" - private String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String FORMAT = "0.1" + private static String METAFOLDERNAME = "meta" + private static String METAFILENAME = "info.json" + private static String METADIFFFILENAME = "differences.txt" + private static String METAARTIFACTS = "${METAFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -507,10 +517,14 @@ class MetaInfo extends GISJsonSerialised { return path } - String metafilepath() { + static String metafilepath() { return METAFOLDERNAME + "/" + METAFILENAME } - + + static String difffilepath() { + return METAFOLDERNAME + "/" + METADIFFFILENAME + } + MetaInfo importrelated(String destpath) { MetaInfo mi = new MetaInfo(script) @@ -523,6 +537,17 @@ class MetaInfo extends GISJsonSerialised { RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) + def importedmetadifffile = adjustedpath + "/" + difffilepath() + if (script.fileExists(importedmetadifffile)) { + def String newcontents = script.readFile(file: importedmetadifffile) + def String currentcontents = "" + if (script.fileExists(difffilepath())) { + currentcontents = script.readFile(file: difffilepath()) + } + + script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) + } + return mi } @@ -563,6 +588,42 @@ class MetaInfo extends GISJsonSerialised { saveJsonSerialised(metafilepath) return artifactlist } + + // Recurse tree of meta looking for release info and accumulate path - Pass by reference unavailable in Groovy/Java + private ArrayList releasewalkerp(int maxdepth, ArrayList releasemetacollection, String pathsofar) { + if (maxdepth > 0) { + if (releaseinfo.releaserepoURL) { + RelatedMetaInfo rmi = new RelatedMetaInfo(script, pathsofar, this) + releasemetacollection.add(rmi) + } + if (pathsofar) { + pathsofar += "/" + } + for (relatedmetainfo in associatedmeta) { + MetaInfo othermeta = relatedmetainfo.meta + String newpathsofar = pathsofar + relatedmetainfo.offsetpath + releasemetacollection = othermeta.releasewalkerp(maxdepth - 1, releasemetacollection, newpathsofar) + } + } + return releasemetacollection + } + + ArrayList releasewalker(int maxdepth, String startpath = "") { + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + return releasemetacollection + } + + Map releasewalkerunique(int maxdepth, String startpath = "") { + Map releasemetauniquecollection = new LinkedHashMap() + ArrayList releasemetacollection = new ArrayList() + releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) + for (relatedmetainfo in releasemetacollection) { + String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetauniquecollection + } } MetaInfo metareset(script) { @@ -575,4 +636,10 @@ MetaInfo metareset(script) { return mi } +MetaInfo metafromfolder(String folder = "") { + MetaInfo buildmeta = new MetaInfo(this) + buildmeta.deserialiseJson("${folder}/${MetaInfo.metafilepath()}") + return buildmeta +} + return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 59fe2a3..a7b583a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -1,15 +1,18 @@ +def int maxversionnumber() { + def int MAXVERSIONNUMBER = 65535 + return MAXVERSIONNUMBER +} + def getGitRevision(String gitrootpath, int basebuildnumber) { def int adjustedchangecount def String gitrevision def String remoteinfo def String remoteurl dir( gitrootpath) { - def int MAXVERSIONNUMBER = 65535 - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (MAXVERSIONNUMBER + 1) + adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() @@ -18,7 +21,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "git remote get-url --push origin").trim() + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] @@ -28,8 +31,6 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { // origin/master // origin/2.0.x def getVersionPrefix(String sourcebranch) { - def int MAXVERSIONNUMBER = 65535 - echo "Determining version number prefix from source branch name: ${sourcebranch}" sourcebranch = sourcebranch.toLowerCase() @@ -65,7 +66,7 @@ def getVersionPrefix(String sourcebranch) { def integers = [] for (integerstring in integerstrings) { int integer = integerstring.toInteger() - integer = integer % (MAXVERSIONNUMBER + 1) + integer = integer % (maxversionnumber() + 1) integers.add(integer) } @@ -112,7 +113,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum echo "Build stamp is: ${buildstamp}" meta.gitinfo.set(sourcepath, sourcerevision, gitrevision, remoteurl, remoteinfo) - meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp, buildstamp.replace(".", ",")) + meta.buildstampinfo.set(versionprefix, gitrevisioncount, buildstamp) return buildstamp } diff --git a/versionnumbering/metaversion.groovy b/versionnumbering/metaversion.groovy deleted file mode 100644 index 44bfbde..0000000 --- a/versionnumbering/metaversion.groovy +++ /dev/null @@ -1,100 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") -def fileutils = load("${autobuilder.commonscripts()}/file/utils.groovy") - -def prefixline(rootpath, filepath) { - def fileContents = readFile("${filepath}").trim() - def fileContentsNormalized = fileContents.normalize() - - def newFileContents = "" - def lines = fileContentsNormalized.readLines() - for (aline in lines) { - newFileContents += "${rootpath}" + "/" + aline + "${nl}" - } - return newFileContents -} - -def mergemeta(rootpath, versionFolder, outputFolder) { - echo "Merging meta data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + newContents - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = prefixline(rootpath, "${buildStampFile}") - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + buildStampFileContents - - def intrevision = 0 - if (fileExists("${versionFolder}/revision.txt")) { - def revision = readFile("${versionFolder}/revision.txt").trim() - intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - } - else { - intrevision = 1 - } - - return intrevision -} - -def appendcomponentinfo(rootpath, versionFolder, outputFolder) { - echo "Appending component version data from folder: ${versionFolder} to ${outputFolder}" - - def differencesFile = "${versionFolder}/differences.txt" - if (fileExists("${differencesFile}")) { - def newContents = readFile file: "${differencesFile}" - def metadifferencesFile = "${outputFolder}/differences.txt" - writeFile file: "${metadifferencesFile}", text: fileutils.currentContents(metadifferencesFile) + "${differencesFile}:\n" + newContents - } - - def buildStampFile = "${versionFolder}/buildstamps.txt" - def buildStampFileContents = readFile("${buildStampFile}").trim() - - def remoteurlFile = "${versionFolder}/remoteurl.txt" - def remoteurlFileContents = fileutils.currentContents(remoteurlFile, "", 'Unknown').trim() - - def gitRevisionFile = "${versionFolder}/gitrevision.txt" - def gitRevisionFileContents = readFile("${gitRevisionFile}").trim() - - def releaseJobNameFile = "${versionFolder}/releasejobname.txt" - def releaseJobNameContents = fileutils.currentContents("${releaseJobNameFile}", "", "-").trim() - - def metabuildStampFile = "${outputFolder}/buildstamps.txt" - writeFile file: "${metabuildStampFile}", text: fileutils.currentContents(metabuildStampFile) + "${rootpath}" + ":" + buildStampFileContents + "," + remoteurlFileContents + "," + gitRevisionFileContents + "," + releaseJobNameContents - def revision = readFile("${versionFolder}/revision.txt").trim() - def intrevision = revision.toInteger() - echo "Revision read in: ${intrevision}" - - return intrevision -} - -// Each version folder should have been created using version creation scripts in this Common_scripts folder -// as their are well-known files in each folder -def combineSubjobVersions(versionLabelPrefix, artifactList, outputFolder = "${WORKSPACE}/metaversion") { - def totalrevision = 0 - dir("${outputFolder}") { - deleteDir() - } - for (artifactItem in artifactList) { - (rootpath, versionFolder) = artifactItem - echo "Inspecting folder: ${versionFolder}" - def metaversionsignaturefile = "${versionFolder}/metaversion.txt" - // If the sub project is a meta job - def intrevision = 0 - if (fileExists("${metaversionsignaturefile}")) { - intrevision = mergemeta(rootpath, versionFolder, outputFolder) - } - else { - intrevision = appendcomponentinfo(rootpath, versionFolder, outputFolder) - } - totalrevision += intrevision - } - echo "Accumulated revision is: ${totalrevision}" - writeFile file: "${outputFolder}/metaversion.txt", text: versionLabelPrefix + ((versionLabelPrefix.endsWith(".")) ? "" : ".") + totalrevision.toString() - writeFile(file: "${outputFolder}/revision.txt", text: totalrevision.toString()) -} - -return this; \ No newline at end of file From 694f633e90568dd994f25c75d02241fb0cce5340 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 15:19:55 +0000 Subject: [PATCH 333/558] Alter differences file generation; Merge differences when importing --- differences/show_release_differences.py | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 80b636e..5c0f15d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -188,6 +188,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: + os.makedirs(os.path.dirname(difffile), exist_ok=True) outfh = open(difffile, 'w') else: outfh = sys.stdout From 74ceec1d875824def70ddd0a0b9a359da227d48b Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Thu, 29 Nov 2018 15:26:07 +0000 Subject: [PATCH 334/558] Merge commit 'aa497116fd9078857227afef34bf88c184030998' into 1.10003.j20-comscr --- differences/show_release_differences.py | 1 + 1 file changed, 1 insertion(+) diff --git a/differences/show_release_differences.py b/differences/show_release_differences.py index 80b636e..5c0f15d 100644 --- a/differences/show_release_differences.py +++ b/differences/show_release_differences.py @@ -188,6 +188,7 @@ USAGE print("Jira key prefix:", jirakeyprefix, "Output options:", outputoptions, "Diff file:", difffile) if difffile: + os.makedirs(os.path.dirname(difffile), exist_ok=True) outfh = open(difffile, 'w') else: outfh = sys.stdout From 49718547b78ed58e900d14f47e9319a17ce9686d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 17:48:05 +0000 Subject: [PATCH 335/558] Move release functionality into Common_scripts --- release/do.groovy | 131 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 release/do.groovy diff --git a/release/do.groovy b/release/do.groovy new file mode 100644 index 0000000..72c2f9f --- /dev/null +++ b/release/do.groovy @@ -0,0 +1,131 @@ +boolean markunstable = false +String unstablereason = "" + +String summaryfolder = "${WORKSPACE}/summary" +String summaryfile = "${summaryfolder}/summary.txt" +String releaserepofolder = "${WORKSPACE}/releaserepos" +String tempfolder = "${WORKSPACE}/temp" + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def start() { + markunstable = false + unstablereason = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary + "\n") +} + +def checkoutreleaserepo(info, String checkoutfolder) { + def String releaserepo = info.meta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component + dir(releaserepofolder) { + sh("git clone -n --depth 1 ${releaserepo} ${checkoutfolder}") + } +} + +def getcomponentartifacts(info, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + for (artifactfilter in info.meta.releaseinfo.releaseitems) { + def filterpath = artifactfilter + if (info.offsetpath) { + filterpath = info.offsetpath + '/' + artifactfilter + } + // Copy relevant artifacts to temp folder + echo "Copying artifacts: ${filterpath} to ${tempfolder}" + copyArtifacts( + filter: filterpath, + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: tempfolder + ) + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + info.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } + summaryappend("${artifactfilter},") + } +} + +def gitrelease(String releaserepo, String buildstamp) { + summaryappend(" Git operations: ") + dir(releaserepo) { + // Add anything new + sh("git add .") + summaryappend("Add,") + // Perform git commit (if necessary) + def changesoccurred = sh(script: "git status --porcelain", returnStdout: true) + if (changesoccurred) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") + } + else { + summaryappend("No-commit,") + } + // Tag with buildstamp + sh("git tag ${buildstamp}") + summaryappend("Tag (${buildstamp}),") + if (DRY_RUN_PUSH == "true") { + sh("git push -n --tags origin master") + summaryappend("Dry-Run-Push,") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend("Push") + } + } +} + +def releasecomponent(info) { + def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${info.meta.productinfo.productname} ${info.meta.productinfo.subcomponentname}: ") + checkoutreleaserepo(info, uniquefolderforrepo) + + getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) + + def String buildstamp = info.meta.buildstampinfo.buildstamp + gitrelease(uniquefolderforrepo, buildstamp) +} + +def releasehierarchy(buildmeta, int maxdepth = MAX_META_DEPTH) { + start() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = buildmeta.releasewalkerunique(maxdepth.toInteger()) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def info = componentReleaseInfo[key] + releasecomponent(info) + } + return [markunstable, unstablereason] +} + +return this; \ No newline at end of file From 94bb1d5c7dbe63d6b598be07f72d55a6b4fb5185 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 17:56:08 +0000 Subject: [PATCH 336/558] Move release functionality into Common_scripts --- release/do.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 72c2f9f..239cc17 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -114,11 +114,12 @@ def releasecomponent(info) { gitrelease(uniquefolderforrepo, buildstamp) } -def releasehierarchy(buildmeta, int maxdepth = MAX_META_DEPTH) { +def releasehierarchy(buildmeta, String maxdepth) { + int md = maxdepth.toInteger() start() // Walk the meta information to a max depth, looking for artifacts to import for release - componentReleaseInfo = buildmeta.releasewalkerunique(maxdepth.toInteger()) + componentReleaseInfo = buildmeta.releasewalkerunique(md) // For each component for (key in componentReleaseInfo.keySet()) { From 24c40a47d951e3ce6244fd3fa048063d4e1e5036 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:09:13 +0000 Subject: [PATCH 337/558] Move release functionality into Common_scripts --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 239cc17..17090fe 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -1,10 +1,10 @@ -boolean markunstable = false -String unstablereason = "" +markunstable = false +unstablereason = "" -String summaryfolder = "${WORKSPACE}/summary" -String summaryfile = "${summaryfolder}/summary.txt" -String releaserepofolder = "${WORKSPACE}/releaserepos" -String tempfolder = "${WORKSPACE}/temp" +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +releaserepofolder = "${WORKSPACE}/releaserepos" +tempfolder = "${WORKSPACE}/temp" def resetfolders() { dir(summaryfolder) { From 3500c13448bb2326bc50dbe39da037778e1b6813 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:11:37 +0000 Subject: [PATCH 338/558] Move release functionality into Common_scripts --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index 17090fe..177593d 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -118,6 +118,7 @@ def releasehierarchy(buildmeta, String maxdepth) { int md = maxdepth.toInteger() start() + resetfolders() // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) From e0f0194df0306a9bf9adb5cc06e119faad3eb0df Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:15:32 +0000 Subject: [PATCH 339/558] Move release functionality into Common_scripts --- release/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 177593d..1eaf2b8 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -18,7 +18,7 @@ def resetfolders() { } } -def start() { +def preconditions() { markunstable = false unstablereason = "" writeFile(file: summaryfile, text: "Release summary: \n") @@ -116,9 +116,9 @@ def releasecomponent(info) { def releasehierarchy(buildmeta, String maxdepth) { int md = maxdepth.toInteger() - start() - resetfolders() + preconditions() + // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) From 6765aaa932221f24fafae8304b03a3ba7adcf97f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:19:59 +0000 Subject: [PATCH 340/558] Move release functionality into Common_scripts --- release/do.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 1eaf2b8..264bd53 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -70,7 +70,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { } } -def gitrelease(String releaserepo, String buildstamp) { +def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { // Add anything new @@ -88,7 +88,7 @@ def gitrelease(String releaserepo, String buildstamp) { // Tag with buildstamp sh("git tag ${buildstamp}") summaryappend("Tag (${buildstamp}),") - if (DRY_RUN_PUSH == "true") { + if (dryrunpush == "true") { sh("git push -n --tags origin master") summaryappend("Dry-Run-Push,") setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") @@ -100,7 +100,7 @@ def gitrelease(String releaserepo, String buildstamp) { } } -def releasecomponent(info) { +def releasecomponent(info, String dryrunpush) { def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -111,10 +111,10 @@ def releasecomponent(info) { getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) def String buildstamp = info.meta.buildstampinfo.buildstamp - gitrelease(uniquefolderforrepo, buildstamp) + gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) } -def releasehierarchy(buildmeta, String maxdepth) { +def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { int md = maxdepth.toInteger() resetfolders() preconditions() @@ -125,7 +125,7 @@ def releasehierarchy(buildmeta, String maxdepth) { // For each component for (key in componentReleaseInfo.keySet()) { def info = componentReleaseInfo[key] - releasecomponent(info) + releasecomponent(info, dryrunpush) } return [markunstable, unstablereason] } From 1e2d7e257a2a81f97e89914cafa2808c29d2d577 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 29 Nov 2018 18:39:53 +0000 Subject: [PATCH 341/558] Move release functionality into Common_scripts --- release/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 264bd53..6db301c 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -31,7 +31,7 @@ def setunstable(String reason) { def summaryappend(String summary) { def String currentsummarycontents = readFile(file: summaryfile) - writeFile(file: summaryfile, text: currentsummarycontents + summary + "\n") + writeFile(file: summaryfile, text: currentsummarycontents + summary) } def checkoutreleaserepo(info, String checkoutfolder) { @@ -112,6 +112,8 @@ def releasecomponent(info, String dryrunpush) { def String buildstamp = info.meta.buildstampinfo.buildstamp gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) + + summaryappend("\n") } def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { From 522ec5c40960c8b6685904a004fff95b8d31a0da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:07:08 +0000 Subject: [PATCH 342/558] Check for changes occurring --- release/do.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 6db301c..0d8e0de 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -73,11 +73,14 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { + def String precommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" // Add anything new sh("git add .") summaryappend("Add,") // Perform git commit (if necessary) - def changesoccurred = sh(script: "git status --porcelain", returnStdout: true) + def changes = sh(script: "git status --porcelain", returnStdout: true) + boolean changesoccurred = changes?.trim() if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") @@ -85,6 +88,8 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { else { summaryappend("No-commit,") } + def String postcommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${precommitshacode}" // Tag with buildstamp sh("git tag ${buildstamp}") summaryappend("Tag (${buildstamp}),") From 2100c7a0f371ecf347aab8b2605d5d1798e6f0c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:09:48 +0000 Subject: [PATCH 343/558] Check for changes occurring --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0d8e0de..cd49d41 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -73,7 +73,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { summaryappend(" Git operations: ") dir(releaserepo) { - def String precommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" // Add anything new sh("git add .") @@ -88,7 +88,7 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { else { summaryappend("No-commit,") } - def String postcommitshacode = sh("git rev-parse --verify HEAD", returnStdout: true).trim() + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${precommitshacode}" // Tag with buildstamp sh("git tag ${buildstamp}") From 5616f9154065a0d26cdbcffce89f497785e31dfe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:28:15 +0000 Subject: [PATCH 344/558] Check for changes occurring --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index cd49d41..e72d6f2 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -81,6 +81,7 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { // Perform git commit (if necessary) def changes = sh(script: "git status --porcelain", returnStdout: true) boolean changesoccurred = changes?.trim() + echo "changesoccurred: ${changesoccurred}" if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") From 9b886717159ba65ec6d32e788ed4e6f7a1aed162 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:32:17 +0000 Subject: [PATCH 345/558] Update --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index e72d6f2..11eb648 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -81,11 +81,11 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { // Perform git commit (if necessary) def changes = sh(script: "git status --porcelain", returnStdout: true) boolean changesoccurred = changes?.trim() - echo "changesoccurred: ${changesoccurred}" + echo "changes: ${changes}\nchangesoccurred: ${changesoccurred}" if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") - } + else { summaryappend("No-commit,") } From 15b9fe77bffd29bfcbbf929d79993f6bab003698 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 11:34:57 +0000 Subject: [PATCH 346/558] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 11eb648..d7f05f7 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -85,7 +85,7 @@ def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { if (changesoccurred) { sh("git commit -a -m ${buildstamp}") summaryappend("Commit,") - + } else { summaryappend("No-commit,") } From 419dc725451a9bef1780d99fda80ea7685f3154b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:06:40 +0000 Subject: [PATCH 347/558] Check for changes occurring --- meta/folder.groovy | 2 +- release/do.groovy | 167 +++++++++++++++++++++++++++++++++------------ 2 files changed, 125 insertions(+), 44 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 9c5367e..cfce68f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -568,7 +568,7 @@ class MetaInfo extends GISJsonSerialised { } - String folder() { + static String folder() { return METAFOLDERNAME } diff --git a/release/do.groovy b/release/do.groovy index d7f05f7..80e10de 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -6,6 +6,8 @@ summaryfile = "${summaryfolder}/summary.txt" releaserepofolder = "${WORKSPACE}/releaserepos" tempfolder = "${WORKSPACE}/temp" +metascript = autobuilder.getmetafolderscript() + def resetfolders() { dir(summaryfolder) { deleteDir() @@ -34,21 +36,36 @@ def summaryappend(String summary) { writeFile(file: summaryfile, text: currentsummarycontents + summary) } -def checkoutreleaserepo(info, String checkoutfolder) { - def String releaserepo = info.meta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component - dir(releaserepofolder) { - sh("git clone -n --depth 1 ${releaserepo} ${checkoutfolder}") +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = new metascript.MetaInfo(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(releasemetafolderpath) } + return previousreleasemeta } -def getcomponentartifacts(info, String repofolder, String tempfolder) { +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = new metascript.MetaInfo(this) + dir(releaserepofolder) { + sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + // Clear all existing files + sh("git rm *") + } + return previousreleasemeta +} + +def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - for (artifactfilter in info.meta.releaseinfo.releaseitems) { + for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter - if (info.offsetpath) { - filterpath = info.offsetpath + '/' + artifactfilter + if (buildinfo.offsetpath) { + filterpath = buildinfo.offsetpath + '/' + artifactfilter } // Copy relevant artifacts to temp folder echo "Copying artifacts: ${filterpath} to ${tempfolder}" @@ -62,7 +79,7 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { target: tempfolder ) // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + info.offsetpath + def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { sh("mv * ${repofolder}") } @@ -70,59 +87,123 @@ def getcomponentartifacts(info, String repofolder, String tempfolder) { } } -def gitrelease(String releaserepo, String buildstamp, String dryrunpush) { +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean changed = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + return changed +} + +def ArrayList maketaglist(String buildstamp, String otherreleasetags) { + ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + taglist.add(othertagitem.trim()) + } + return taglist +} + +def boolean releasegittag(String postcommitshacode, String tag) { + // Tag + sh("git tag ${tag}") + summaryappend("(${tag}),") +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend("Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend("Push") + } +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") + def String buildstamp = buildinfo.meta.buildstampinfo.buildstamp dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" - // Add anything new - sh("git add .") - summaryappend("Add,") - // Perform git commit (if necessary) - def changes = sh(script: "git status --porcelain", returnStdout: true) - boolean changesoccurred = changes?.trim() - echo "changes: ${changes}\nchangesoccurred: ${changesoccurred}" - if (changesoccurred) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") + + releasegitadd() + + def boolean changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamp) } else { + releasegitreset() summaryappend("No-commit,") } + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${precommitshacode}" - // Tag with buildstamp - sh("git tag ${buildstamp}") - summaryappend("Tag (${buildstamp}),") - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend("Dry-Run-Push,") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend("Push") + echo "Post commit sha code: ${postcommitshacode}" + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + + def boolean tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) } } } -def releasecomponent(info, String dryrunpush) { - def String uniquefolder = "${info.meta.productinfo.productname}_${info.meta.productinfo.subcomponentname}" +def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { + def buildmeta = buildinfo.meta + def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" - summaryappend("${info.meta.productinfo.productname} ${info.meta.productinfo.subcomponentname}: ") - checkoutreleaserepo(info, uniquefolderforrepo) + summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") + def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) - getcomponentartifacts(info, uniquefolderforrepo, uniquefoldertemp) + getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) - def String buildstamp = info.meta.buildstampinfo.buildstamp - gitrelease(uniquefolderforrepo, buildstamp, dryrunpush) + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, otherreleasetags, dryrunpush) summaryappend("\n") } -def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { +def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String dryrunpush) { int md = maxdepth.toInteger() resetfolders() preconditions() @@ -132,8 +213,8 @@ def releasehierarchy(buildmeta, String maxdepth, dryrunpush) { // For each component for (key in componentReleaseInfo.keySet()) { - def info = componentReleaseInfo[key] - releasecomponent(info, dryrunpush) + def buildinfo = componentReleaseInfo[key] + releasecomponent(buildinfo, otherreleasetags, dryrunpush) } return [markunstable, unstablereason] } From d449e458f29e636a7f9a343237aa6f95ab3703d6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:16:10 +0000 Subject: [PATCH 348/558] Check for changes occurring --- meta/folder.groovy | 4 ++++ release/do.groovy | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index cfce68f..13b1fe5 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -624,6 +624,10 @@ class MetaInfo extends GISJsonSerialised { } return releasemetauniquecollection } + + MetaInfo getNewMetaObject(script) { + return new MetaInfo(script) + } } MetaInfo metareset(script) { diff --git a/release/do.groovy b/release/do.groovy index 80e10de..9d3e09a 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -37,7 +37,7 @@ def summaryappend(String summary) { } def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = new metascript.MetaInfo(this) + def previousreleasemeta = buildmeta.getNewMetaObject(this) // See if there is meta data file from the current release def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" if (fileExists(releasemetafolderpath)) { @@ -49,7 +49,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) { def checkoutreleaserepo(buildmeta, String checkoutfolder) { def String releaserepo = buildmeta.releaseinfo.releaserepoURL // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - def previousreleasemeta = new metascript.MetaInfo(this) + def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) From ef90356389a24a5ebb87cb4d213bb81a0080fb8a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:21:49 +0000 Subject: [PATCH 349/558] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 9d3e09a..576a188 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -41,7 +41,7 @@ def getpreviousreleasemeta(buildmeta, String checkoutfolder) { // See if there is meta data file from the current release def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(releasemetafolderpath) + previousreleasemeta = metascript.metafromfolder(checkoutfolder) } return previousreleasemeta } From d16e5a426a78fd8091f8c274b5b1041130471580 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:27:03 +0000 Subject: [PATCH 350/558] Check for changes occurring --- release/do.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 576a188..644b995 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -52,7 +52,9 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def previousreleasemeta = buildmeta.getNewMetaObject(this) dir(releaserepofolder) { sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { // Clear all existing files sh("git rm *") } From ad9b3482b0a0d34d41ec1df60c9f49484ef6a61e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:29:48 +0000 Subject: [PATCH 351/558] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 644b995..6cf55e2 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -56,7 +56,7 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) dir(checkoutfolder) { // Clear all existing files - sh("git rm *") + sh("git rm -r *") } return previousreleasemeta } From 0388e87adf98c28587d6e60f33750b704957ef7c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 15:32:39 +0000 Subject: [PATCH 352/558] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 6cf55e2..bcdc419 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -160,7 +160,7 @@ def releasegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildinfo.meta.buildstampinfo.buildstamp + def String buildstamp = buildmeta.buildstampinfo.buildstamp dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From e82f6be42017371d3622a9428f987c40f9858b77 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 16:15:58 +0000 Subject: [PATCH 353/558] Check for changes occurring --- release/do.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release/do.groovy b/release/do.groovy index bcdc419..a86bf04 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -129,6 +129,11 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { } def boolean releasegittag(String postcommitshacode, String tag) { + // Check if tag exists already + def retvalue = sh(script: "git rev-parse ${tag} >2 /dev/null", returnStdout: true, returnStatus: true) + echo "retvalue: ${retvalue} class: ${retvalue.getClass()}" + + // Tag sh("git tag ${tag}") summaryappend("(${tag}),") From 02e759acbf187f12dcc9d79b4da9b2d76587f1b2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 16:38:43 +0000 Subject: [PATCH 354/558] Check for changes occurring --- release/do.groovy | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a86bf04..2438b54 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -129,14 +129,26 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { } def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false // Check if tag exists already - def retvalue = sh(script: "git rev-parse ${tag} >2 /dev/null", returnStdout: true, returnStatus: true) - echo "retvalue: ${retvalue} class: ${retvalue.getClass()}" - - - // Tag - sh("git tag ${tag}") - summaryappend("(${tag}),") + def existingshacode = sh(script: "git rev-parse ${tag}", returnStdout: true).trim() + if (existingshacode?.trim()) { // If tag/sha code does exist + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!),") + } + else { + summaryappend("(${tag} - Already added),") + } + } + else { // If sha code does not exist already + // Tag + sh("git tag ${tag}") + summaryappend("(${tag}),") + tagadded = true + } + return tagadded } def boolean releasegittags(String postcommitshacode, ArrayList taglist) { From a5703babe06575ffd5b9d745e81ca6c066fd5b83 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 16:48:17 +0000 Subject: [PATCH 355/558] Check for changes occurring --- release/do.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2438b54..e69f09b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -125,6 +125,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } + echo "Taglist: ${taglist}" return taglist } @@ -136,16 +137,16 @@ def boolean releasegittag(String postcommitshacode, String tag) { if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!),") + summaryappend("(${tag} - Could not add!)") } else { - summaryappend("(${tag} - Already added),") + summaryappend("(${tag} - Already added)") } } else { // If sha code does not exist already // Tag sh("git tag ${tag}") - summaryappend("(${tag}),") + summaryappend("(${tag})") tagadded = true } return tagadded From 16a7f88ec3ed498f5044f41de685add319a4f931 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Fri, 30 Nov 2018 16:53:54 +0000 Subject: [PATCH 356/558] ELGIP-20 let script run on windows --- versionnumbering/generate.groovy | 33 ++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index a7b583a..803a543 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -10,18 +10,43 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + + if (isUnix()) { + def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() + def int changecount = stdout.readLines()[1] as int + } echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" - gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + if (isUnix()) { + gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git rev-parse HEAD").trim() + gitrevision = stdout.readLines()[1] as int + } echo "Revision: ${gitrevision}" - remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + if (isUnix()) { + remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git remote show origin").trim() + remoteinfo = stdout.readLines()[1] as int + } echo "Remote info: ${remoteinfo}" - remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + if (isUnix()) { + remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() + } + else { + def stdout = bat(returnStdout:true , script: "git config --get remote.origin.ur").trim() + remoteurl = stdout.readLines()[1] as int + } echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From c1553971f3caac2fbff4cf097f675469093cddc9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:16:45 +0000 Subject: [PATCH 357/558] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index e69f09b..1fe7cd4 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -125,7 +125,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } - echo "Taglist: ${taglist}" + echo "Taglist: ${taglist} size: ${taglist.size()}" return taglist } From 460b758b05b9fe51d3e24c2f5beaa12b0a83a07b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:20:36 +0000 Subject: [PATCH 358/558] Check for changes occurring --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1fe7cd4..788838b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -121,7 +121,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { ArrayList taglist = new ArrayList() taglist.add(buildstamp) - ArrayList othertaglist = otherreleasetags.split(',') + ArrayList othertaglist = otherreleasetags.trim().split(',') for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } From 2dcb050ab38355b30d39f1f2a0032354b99866fe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:36:43 +0000 Subject: [PATCH 359/558] Check for changes occurring --- release/do.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 788838b..5d4f2e9 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -119,9 +119,12 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { def ArrayList maketaglist(String buildstamp, String otherreleasetags) { ArrayList taglist = new ArrayList() - taglist.add(buildstamp) + taglist.add(buildstamp.trim()) + echo "Taglist: ${taglist} size: ${taglist.size()}" + ArrayList othertaglist = otherreleasetags.trim().split(',') + echo "othertaglist: ${othertaglist} size: ${othertaglist.size()}" for (othertagitem in othertaglist) { taglist.add(othertagitem.trim()) } @@ -178,7 +181,7 @@ def releasegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp + def String buildstamp = buildmeta.buildstampinfo.buildstamp.trim() dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From 55262543d73eea07a265a815db213e360a8a9525 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 17:41:40 +0000 Subject: [PATCH 360/558] Check for changes occurring --- release/do.groovy | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 5d4f2e9..7b7c07c 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -118,15 +118,14 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { } def ArrayList maketaglist(String buildstamp, String otherreleasetags) { - ArrayList taglist = new ArrayList() - taglist.add(buildstamp.trim()) + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) - echo "Taglist: ${taglist} size: ${taglist.size()}" - - ArrayList othertaglist = otherreleasetags.trim().split(',') - echo "othertaglist: ${othertaglist} size: ${othertaglist.size()}" + def ArrayList othertaglist = otherreleasetags.split(',') for (othertagitem in othertaglist) { - taglist.add(othertagitem.trim()) + if (othertagitem?.trim()) { + taglist.add(othertagitem.trim()) + } } echo "Taglist: ${taglist} size: ${taglist.size()}" return taglist @@ -181,7 +180,7 @@ def releasegitpush(String dryrunpush ) { def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") - def String buildstamp = buildmeta.buildstampinfo.buildstamp.trim() + def String buildstamp = buildmeta.buildstampinfo.buildstamp dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From b8a0e604f1d84a4d1e1cbdc52f04db831b67f776 Mon Sep 17 00:00:00 2001 From: Damian Smith Date: Fri, 30 Nov 2018 17:55:55 +0000 Subject: [PATCH 361/558] ELGIP-20 edits to allow use on windows --- versionnumbering/generate.groovy | 56 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 803a543..925ed65 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -10,43 +10,43 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { def String remoteinfo def String remoteurl dir( gitrootpath) { - - if (isUnix()) { - def int changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + def int changecount + if (isUnix()) { + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() } - else { - def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() - def int changecount = stdout.readLines()[1] as int - } + else { + def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() + changecount = stdout.readLines()[1] as int + } echo "Change count: ${changecount}" adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" - - if (isUnix()) { + + if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() - } - else { - def stdout = bat(returnStdout:true , script: "git rev-parse HEAD").trim() - gitrevision = stdout.readLines()[1] as int - } + } + else { + def stdout = bat(returnStdout:true , script: "git rev-parse HEAD").trim() + gitrevision = stdout.readLines()[1] + } echo "Revision: ${gitrevision}" - - if (isUnix()) { + + if (isUnix()) { remoteinfo = sh(returnStdout: true, script: "git remote show origin").trim() - } - else { - def stdout = bat(returnStdout:true , script: "git remote show origin").trim() - remoteinfo = stdout.readLines()[1] as int - } + } + else { + def stdout = bat(returnStdout:true , script: "git remote show origin").trim() + remoteinfo = stdout.readLines()[1..-1] + } echo "Remote info: ${remoteinfo}" - - if (isUnix()) { + + if (isUnix()) { remoteurl = sh(returnStdout: true, script: "git config --get remote.origin.url").trim() - } - else { - def stdout = bat(returnStdout:true , script: "git config --get remote.origin.ur").trim() - remoteurl = stdout.readLines()[1] as int - } + } + else { + def stdout = bat(returnStdout:true , script: "git config --get remote.origin.url").trim() + remoteurl = stdout.readLines()[1] + } echo "Remote URL: ${remoteurl}" } return [adjustedchangecount, gitrevision, remoteinfo, remoteurl] From c4b875e5e9fe925d8da100789d1d05719399bd4f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 30 Nov 2018 18:46:33 +0000 Subject: [PATCH 362/558] Check for changes occurring --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 7b7c07c..b2f09bd 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -127,7 +127,6 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { taglist.add(othertagitem.trim()) } } - echo "Taglist: ${taglist} size: ${taglist.size()}" return taglist } From 9b5c42f42824398390ac7730e3bd130fd0a0def9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 08:25:26 +0000 Subject: [PATCH 363/558] Make sure changecount is an integer --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 925ed65..f80d021 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -12,7 +12,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { dir( gitrootpath) { def int changecount if (isUnix()) { - changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim() + changecount = sh(returnStdout: true, script: "git rev-list HEAD --count").trim().toInteger() } else { def stdout = bat(returnStdout:true , script: "git rev-list HEAD --count").trim() From 09bf96a4b7a89e47229fbd01295c06144c228b6c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 09:32:23 +0000 Subject: [PATCH 364/558] Add a summary on the console --- release/do.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index b2f09bd..2866793 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -5,6 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" releaserepofolder = "${WORKSPACE}/releaserepos" tempfolder = "${WORKSPACE}/temp" +summarystring = "" metascript = autobuilder.getmetafolderscript() @@ -23,6 +24,7 @@ def resetfolders() { def preconditions() { markunstable = false unstablereason = "" + summarystring = "" writeFile(file: summaryfile, text: "Release summary: \n") } @@ -34,6 +36,7 @@ def setunstable(String reason) { def summaryappend(String summary) { def String currentsummarycontents = readFile(file: summaryfile) writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary } def getpreviousreleasemeta(buildmeta, String checkoutfolder) { @@ -237,7 +240,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String def buildinfo = componentReleaseInfo[key] releasecomponent(buildinfo, otherreleasetags, dryrunpush) } - return [markunstable, unstablereason] + return [summarystring, markunstable, unstablereason] } return this; \ No newline at end of file From 3e8f5dbaedc66569acacff5d20d9f8bea7631914 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:19:27 +0000 Subject: [PATCH 365/558] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 2866793..187fb01 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -3,9 +3,10 @@ unstablereason = "" summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" tempfolder = "${WORKSPACE}/temp" -summarystring = "" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" metascript = autobuilder.getmetafolderscript() @@ -136,8 +137,9 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def existingshacode = sh(script: "git rev-parse ${tag}", returnStdout: true).trim() - if (existingshacode?.trim()) { // If tag/sha code does exist + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true).trim() + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD // This is a problem - The tag is already at another position in git repo setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") @@ -147,9 +149,9 @@ def boolean releasegittag(String postcommitshacode, String tag) { summaryappend("(${tag} - Already added)") } } - else { // If sha code does not exist already + else { // If tag does not exist already // Tag - sh("git tag ${tag}") + sh("git tag \"${tag}\"") summaryappend("(${tag})") tagadded = true } @@ -243,4 +245,4 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String return [summarystring, markunstable, unstablereason] } -return this; \ No newline at end of file +return this; From b6007e252c7d9f2746dc84b313ec33505f7f817b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:25:10 +0000 Subject: [PATCH 366/558] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 187fb01..a43beda 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -137,7 +137,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true).trim() + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD From a0c6fc46211e2bebdd525d8240c4012fbb6649f1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:43:26 +0000 Subject: [PATCH 367/558] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a43beda..dc834ea 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -137,7 +137,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\"", returnStatus: true) + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\" 2> /dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD @@ -173,12 +173,12 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) def releasegitpush(String dryrunpush ) { if (dryrunpush == "true") { sh("git push -n --tags origin master") - summaryappend("Dry-Run-Push") + summaryappend(",Dry-Run-Push") setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") } else { sh("git push --tags origin master") - summaryappend("Push") + summaryappend(",Push") } } From 3665d5e9a653a0dfc4445ae676ed9e067e8dfbf7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 10:47:30 +0000 Subject: [PATCH 368/558] Handle case of tag not already existing in release repo correctly --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index dc834ea..f71c3df 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -137,7 +137,7 @@ def ArrayList maketaglist(String buildstamp, String otherreleasetags) { def boolean releasegittag(String postcommitshacode, String tag) { def boolean tagadded = false // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" > \"${gitrevparseoutputfile}\" 2> /dev/null", returnStatus: true) + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) if (gitrevparsestatus == 0) { // Tag exists def existingshacode = readFile(file: gitrevparseoutputfile).trim() if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD From 813d6c77a69e11c18b5d6ed0b5d0fe467f9492a9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 12:27:00 +0000 Subject: [PATCH 369/558] Tag associated source repos --- release/do.groovy | 86 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index f71c3df..a320775 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -5,6 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" @@ -65,6 +66,14 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { return previousreleasemeta } +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + dir(sourcerepofolder) { + sh("git clone --depth 1 ${sourcerepo} ${checkoutfolder}") + } +} + def getcomponentartifacts(buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -112,13 +121,13 @@ def releasegitcommit(String buildstamp) { } // Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { +def checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision def boolean changed = (buildsha != previoussha) echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return changed + return [changed, buildsha] } def ArrayList maketaglist(String buildstamp, String otherreleasetags) { @@ -170,6 +179,42 @@ def boolean releasegittags(String postcommitshacode, ArrayList taglist) return tagadded } +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(buildsha, taglist) { + boolean tagadded = false + + summaryappend("Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + def releasegitpush(String dryrunpush ) { if (dryrunpush == "true") { sh("git push -n --tags origin master") @@ -182,16 +227,34 @@ def releasegitpush(String dryrunpush ) { } } -def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherreleasetags, String dryrunpush) { +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Push") + } +} + +def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String sourcerepo, String otherreleasetags, String dryrunpush) { summaryappend(" Git operations: ") def String buildstamp = buildmeta.buildstampinfo.buildstamp + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + + def boolean changed = false + def String buildsha = "" + def boolean tagadded = false dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" releasegitadd() - def boolean changed = checkforchange(buildmeta, previousreleasemeta) + (changed, buildsha)= checkforchange(buildmeta, previousreleasemeta) if (changed) { releasegitcommit(buildstamp) } @@ -203,14 +266,18 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String otherr def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - - def boolean tagadded = releasegittags(postcommitshacode, taglist) + tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { releasegitpush(dryrunpush) } } + dir(sourcerepo) { + tagadded = sourcegittags(buildsha, taglist) + if (tagadded) { + sourcegitpush(dryrunpush) + } + } } def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { @@ -218,13 +285,16 @@ def releasecomponent(buildinfo, String otherreleasetags, String dryrunpush) { def String uniquefolder = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + def String uniquefolderforsourcerepo = "${sourcerepofolder}/${uniquefolder}" summaryappend("${buildmeta.productinfo.productname} ${buildmeta.productinfo.subcomponentname}: ") def previousreleasemeta = checkoutreleaserepo(buildmeta, uniquefolderforrepo) + checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) + getcomponentartifacts(buildinfo, uniquefolderforrepo, uniquefoldertemp) - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, otherreleasetags, dryrunpush) + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) summaryappend("\n") } From deb48be9fe1bccc9d02d7b57c65e7de999b67b12 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 13:25:00 +0000 Subject: [PATCH 370/558] Tag associated source repos --- release/do.groovy | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index a320775..6992f81 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -121,13 +121,13 @@ def releasegitcommit(String buildstamp) { } // Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def checkforchange(buildmeta, previousreleasemeta) { +def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision def boolean changed = (buildsha != previoussha) echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return [changed, buildsha] + return changed } def ArrayList maketaglist(String buildstamp, String otherreleasetags) { @@ -196,17 +196,17 @@ def boolean sourcegittag(String shacode, String tag) { } else { // If tag does not exist already // Tag - sh("git tag \"${tag}\"") + sh("git tag \"${tag}\" ${shacode}") summaryappend("(${tag})") tagadded = true } return tagadded } -def sourcegittags(buildsha, taglist) { +def sourcegittags(String buildsha, List taglist) { boolean tagadded = false - summaryappend("Src Tag ") + summaryappend(",Src Tag ") for (tag in taglist) { if (sourcegittag(buildsha, tag)) { tagadded = true @@ -243,18 +243,14 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source summaryappend(" Git operations: ") def String buildstamp = buildmeta.buildstampinfo.buildstamp - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - - def boolean changed = false def String buildsha = "" - def boolean tagadded = false dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" releasegitadd() - (changed, buildsha)= checkforchange(buildmeta, previousreleasemeta) + def boolean changed = checkforchange(buildmeta, previousreleasemeta) if (changed) { releasegitcommit(buildstamp) } @@ -266,14 +262,19 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Post commit sha code: ${postcommitshacode}" - tagadded = releasegittags(postcommitshacode, taglist) + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) if (changed || tagadded) { releasegitpush(dryrunpush) } } dir(sourcerepo) { - tagadded = sourcegittags(buildsha, taglist) + def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.buildstampinfo.buildstamp}" + def String buildsha = buildmeta.gitinfo.gitrevision + + def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) + def tagadded = sourcegittags(buildsha, srctaglist) if (tagadded) { sourcegitpush(dryrunpush) } @@ -306,7 +307,7 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) - + // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] From 75fc183c8908baac52fd9678cefec9f8adffe1c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 13:27:12 +0000 Subject: [PATCH 371/558] Tag associated source repos --- release/do.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 6992f81..ce2aef0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -243,7 +243,6 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source summaryappend(" Git operations: ") def String buildstamp = buildmeta.buildstampinfo.buildstamp - def String buildsha = "" dir(releaserepo) { def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() echo "Pre commit sha code: ${precommitshacode}" From ad6e69a19a0439038b8388fad9e69069afa9ce94 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 14:19:59 +0000 Subject: [PATCH 372/558] Tag associated source repos --- release/do.groovy | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index ce2aef0..4a00491 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -69,8 +69,20 @@ def checkoutreleaserepo(buildmeta, String checkoutfolder) { def checkoutsourcerepo(buildmeta, String checkoutfolder) { def String sourcerepo = buildmeta.gitinfo.remoteurl // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - dir(sourcerepofolder) { - sh("git clone --depth 1 ${sourcerepo} ${checkoutfolder}") + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard" + git clean -fdx + git pull + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } } } @@ -269,7 +281,7 @@ def gitrelease(buildmeta, previousreleasemeta, String releaserepo, String source } } dir(sourcerepo) { - def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.buildstampinfo.buildstamp}" + def String srcbuildstamp = "${buildmeta.productinfo.productname}_${buildmeta.productinfo.subcomponentname}_${buildmeta.buildstampinfo.buildstamp}" def String buildsha = buildmeta.gitinfo.gitrevision def ArrayList srctaglist = maketaglist(srcbuildstamp, otherreleasetags) From 2b15ebbd1890f76f60b690fc118c905548425876 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 14:22:22 +0000 Subject: [PATCH 373/558] Tag associated source repos --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 4a00491..f0c6c8b 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -72,7 +72,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { if (fileExists(checkoutfolder)) { dir(checkoutfolder) { sh(""" - git reset --hard" + git reset --hard git clean -fdx git pull """ From f4688da0078abecb9c4ba94bd4ec8a46b0d7c340 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 14:52:01 +0000 Subject: [PATCH 374/558] Make sure product name and sub component names can be used in tags --- meta/folder.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 13b1fe5..7ab5744 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -31,6 +31,15 @@ class ProductInfo extends GISJsonSerialised { } void set(String productname, String subcomponentname, String jirakey) { + // Make sure the fields can be used in git tags + def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\"", , returnStatus: true) + if (pstatus > 0) { + script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") + } + def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\"", , returnStatus: true) + if (sstatus > 0) { + script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") + } this.productname = productname this.subcomponentname = subcomponentname this.jirakey = jirakey From 5e7a447a1032c86e5deda0bd4a7b243ba1c0cc3e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 15:00:02 +0000 Subject: [PATCH 375/558] Make sure product name and sub component names can be used in tags --- meta/folder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 7ab5744..e1e38aa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -32,11 +32,11 @@ class ProductInfo extends GISJsonSerialised { void set(String productname, String subcomponentname, String jirakey) { // Make sure the fields can be used in git tags - def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\"", , returnStatus: true) + def int pstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${productname}\" >/dev/null 2>/dev/null", , returnStatus: true) if (pstatus > 0) { script.error("The product name: ${productname} needs to be renamed as it cannot be used in a git tag name") } - def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\"", , returnStatus: true) + def int sstatus = script.sh(script: "git check-ref-format --normalize \"refs/tags/${subcomponentname}\" >/dev/null 2>/dev/null", , returnStatus: true) if (sstatus > 0) { script.error("The subcomponent name: ${subcomponentname} needs to be renamed as it cannot be used in a git tag name") } From 09984a65b6a6790ac515ba860d4ec8143d6b2062 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 23:27:47 +0000 Subject: [PATCH 376/558] Make sure all excess local tags are removed from the source repo at the start --- release/do.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/release/do.groovy b/release/do.groovy index f0c6c8b..ecdd4eb 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -75,6 +75,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { git reset --hard git clean -fdx git pull + # Remove all tags that are not already on the remote + git fetch --tags --prune """ ) } From 5a60dddd9a4e55a4ce5e30e21724dd9c16a4ec1e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 1 Dec 2018 23:53:17 +0000 Subject: [PATCH 377/558] Make sure all excess local tags are removed from the source repo at the start --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index ecdd4eb..c8e1ce6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -76,7 +76,8 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { git clean -fdx git pull # Remove all tags that are not already on the remote - git fetch --tags --prune + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null """ ) } From 299f74fb853c207b19b018cec2cf5ca7bad61981 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 2 Dec 2018 17:04:33 +0000 Subject: [PATCH 378/558] Distinguish src and release repo pushes --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index c8e1ce6..169c292 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -245,12 +245,12 @@ def releasegitpush(String dryrunpush ) { def sourcegitpush(String dryrunpush ) { if (dryrunpush == "true") { sh("git push -n --tags") - summaryappend(",Dry-Run-Push") + summaryappend(",Src-Dry-Run-Push") setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") } else { sh("git push --tags") - summaryappend(",Push") + summaryappend(",Src Push") } } From 783cb9c9629fb5275ec3e7d666bfc9d6f1e16d65 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 2 Dec 2018 18:08:43 +0000 Subject: [PATCH 379/558] Make sure product name and sub component names can be used in tags --- meta/folder.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index e1e38aa..1edd45c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -434,6 +434,7 @@ class MetaInfo extends GISJsonSerialised { ChecksumInfo checksuminfo private ArrayList configurationinfo private ArrayList publishableinfo + private LinkedHashMap custominfo private ArrayList associatedmeta MetaInfo(script) { @@ -446,6 +447,7 @@ class MetaInfo extends GISJsonSerialised { checksuminfo = new ChecksumInfo(script) configurationinfo = new ArrayList() publishableinfo = new ArrayList() + custominfo = new LinkedHashMap() associatedmeta = new ArrayList() } @@ -467,6 +469,14 @@ class MetaInfo extends GISJsonSerialised { configurationinfo.add(ci) } + void addCustomInfo(String customId, String customValue) { + custominfo.put(customId.trim(), customValue) + } + + String getCustomInfo(String customId) { + return custominfo.getAt(customId.trim()) + } + List publishableinfoserialised() { def publishable = [] for (publishableitem in publishableinfo) { @@ -504,6 +514,7 @@ class MetaInfo extends GISJsonSerialised { pb.deserialiseJson(rawpublishable) publishableinfo.add(pb) } + custominfo = rawmeta['custominfo'] for (rawrelatedmeta in rawmeta['associatedmeta']) { def rm = new RelatedMetaInfo(script) rm.deserialiseJson(rawrelatedmeta) @@ -571,6 +582,7 @@ class MetaInfo extends GISJsonSerialised { 'configuration' : configurationinfoserialised(), 'checksuminfo' : checksuminfo.serialisationdata(), 'publishableinfo' : publishableinfoserialised(), + 'custominfo' : custominfo, 'associatedmeta' : associatedmetaserialised(), ] return data From 165888b6d98a6eaa5ddf9b32947eb7dc7407b388 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Dec 2018 11:03:43 +0000 Subject: [PATCH 380/558] Turn off fingerprinting --- install/installhelper.gsh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.gsh b/install/installhelper.gsh index a55292d..c1ccc6a 100644 --- a/install/installhelper.gsh +++ b/install/installhelper.gsh @@ -9,7 +9,7 @@ def artifactsImporter(artifactDetails, installerDetails) { env.importDest = destFolder if (artifactFetched.get(artifactKey) == null) { //echo "Artifact info: ${artifactInfo}" - copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: true, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" + copyArtifacts excludes: notToCopy, filter: toCopy, fingerprintArtifacts: false, projectName: "${sourceProject}", selector: lastSuccessful(), target: "${destFolder}" def componentVersionFolder = "${WORKSPACE}\\version\\version_${artifactKey}" def folderToCopy = "${destFolder}\\version" echo "Copying from: ${folderToCopy} to: ${componentVersionFolder}" From c53ab475b15f335709421f54a291b8b925351c6b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 10 Dec 2018 18:30:15 +0000 Subject: [PATCH 381/558] Add publishing script --- publish/do.groovy | 313 ++++++++++++++++++++++++++++++++++++++ versioncontrol/git.groovy | 18 ++- 2 files changed, 329 insertions(+), 2 deletions(-) create mode 100644 publish/do.groovy diff --git a/publish/do.groovy b/publish/do.groovy new file mode 100644 index 0000000..2c5a018 --- /dev/null +++ b/publish/do.groovy @@ -0,0 +1,313 @@ +markunstable = false +unstablereason = "" + +FTP_CLIENT_LOCATION='ftp' +FTP_URL='ie1.hostedftp.com' +FTP_USER='ftp@globalinkjetsystems.com' +FTP_CUSTOMER_ROOT='/Customers/' +FTP_PASSWORD='ramshorn42' +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Publish summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason = reason +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def getpreviousreleasemeta(buildmeta, String checkoutfolder) { + def previousreleasemeta = buildmeta.getNewMetaObject(this) + // See if there is meta data file from the current release + def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" + if (fileExists(releasemetafolderpath)) { + previousreleasemeta = metascript.metafromfolder(checkoutfolder) + } + return previousreleasemeta +} + +def checkoutreleaserepo(buildmeta, String checkoutfolder) { + def String releaserepo = buildmeta.releaseinfo.releaserepoURL + // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta + def previousreleasemeta = buildmeta.getNewMetaObject(this) + dir(releaserepofolder) { + sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") + } + previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) + dir(checkoutfolder) { + // Clear all existing files + sh("git rm -r *") + } + return previousreleasemeta +} + +def checkoutsourcerepo(buildmeta, String checkoutfolder) { + def String sourcerepo = buildmeta.gitinfo.remoteurl + // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component + if (fileExists(checkoutfolder)) { + dir(checkoutfolder) { + sh(""" + git reset --hard + git clean -fdx + git pull + # Remove all tags that are not already on the remote + git tag -l | xargs git tag -d >/dev/null + git fetch -t >/dev/null + """ + ) + } + } + else { + dir(sourcerepofolder) { + sh("git clone ${sourcerepo} ${checkoutfolder}") + } + } +} + +def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { + // Copy relevant artifacts to temp folder + summaryappend("Artifacts: ") + + def releasemeta = releaseinfo.meta + def String componentURL = releasemeta.releaseinfo.releaserepoURL + def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" + def publishableitems = releasemeta.publishableinfo + def ArrayList publishablePaths = new ArrayList() + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone +} + +def releasegitadd() { + // Add anything new + sh("git add .") + summaryappend("Add,") +} + +def releasegitreset() { + // Add anything new + sh("git reset --hard") + sh("git clean -fdx") + summaryappend("Reset,") +} + +def releasegitcommit(String buildstamp) { + sh("git commit -a -m ${buildstamp}") + summaryappend("Commit,") +} + +// Determine if there has been a change by comparing the sha code in previous release meta and the build meta +def boolean checkforchange(buildmeta, previousreleasemeta) { + String buildsha = buildmeta.gitinfo.gitrevision + String previoussha = previousreleasemeta.gitinfo.gitrevision + + def boolean changed = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + return changed +} + +def ArrayList maketaglist(String buildstamp, String otherreleasetags) { + def ArrayList taglist = new ArrayList() + taglist.add(buildstamp) + + def ArrayList othertaglist = otherreleasetags.split(',') + for (othertagitem in othertaglist) { + if (othertagitem?.trim()) { + taglist.add(othertagitem.trim()) + } + } + return taglist +} + +def boolean releasegittag(String postcommitshacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\"") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def boolean releasegittags(String postcommitshacode, ArrayList taglist) { + boolean tagadded = false + + summaryappend("Rel Tag ") + for (tag in taglist) { + if (releasegittag(postcommitshacode, tag)) { + tagadded = true + } + } + return tagadded +} + +def boolean sourcegittag(String shacode, String tag) { + def boolean tagadded = false + // Check if tag exists already + def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) + if (gitrevparsestatus == 0) { // Tag exists + def existingshacode = readFile(file: gitrevparseoutputfile).trim() + if (existingshacode != shacode) { // If the existing tag is not associated with HEAD + // This is a problem - The tag is already at another position in git repo + setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") + summaryappend("(${tag} - Could not add!)") + } + else { + summaryappend("(${tag} - Already added)") + } + } + else { // If tag does not exist already + // Tag + sh("git tag \"${tag}\" ${shacode}") + summaryappend("(${tag})") + tagadded = true + } + return tagadded +} + +def sourcegittags(String buildsha, List taglist) { + boolean tagadded = false + + summaryappend(",Src Tag ") + for (tag in taglist) { + if (sourcegittag(buildsha, tag)) { + tagadded = true + } + } + return tagadded +} + +def releasegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags origin master") + summaryappend(",Dry-Run-Push") + setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags origin master") + summaryappend(",Push") + } +} + +def sourcegitpush(String dryrunpush ) { + if (dryrunpush == "true") { + sh("git push -n --tags") + summaryappend(",Src-Dry-Run-Push") + setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") + } + else { + sh("git push --tags") + summaryappend(",Src Push") + } +} + +def publish(releasemeta, String uniquefolderforrepo) { + summaryappend(" Git operations: ") + def String buildstamp = releasemeta.buildstampinfo.buildstamp + + dir(uniquefolderforrepo) { + def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Pre commit sha code: ${precommitshacode}" + + releasegitadd() + + def boolean changed = checkforchange(buildmeta, previousreleasemeta) + if (changed) { + releasegitcommit(buildstamp) + } + else { + releasegitreset() + summaryappend("No-commit,") + } + + def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() + echo "Post commit sha code: ${postcommitshacode}" + + ArrayList taglist = maketaglist(buildstamp, otherreleasetags) + def tagadded = releasegittags(postcommitshacode, taglist) + + if (changed || tagadded) { + releasegitpush(dryrunpush) + } + } +} + +def recallcomponent(releaseinfo) { + def releasemeta = releaseinfo.meta + def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" + def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" + + summaryappend("${releasemeta.productinfo.productname} ${releasemeta.productinfo.subcomponentname}: ") + + getcomponentartifacts(releaseinfo, uniquefolderforrepo, uniquefoldertemp) + + summaryappend("\n") +} + +def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for artifacts to import for release + componentReleaseInfo = releasemeta.releasewalkerunique(md) + + // For each component + for (key in componentReleaseInfo.keySet()) { + def releaseinfo = componentReleaseInfo[key] + recallcomponent(releaseinfo) + } + // publish(releasemeta, uniquefolderforrepo) + + return [summarystring, markunstable, unstablereason] +} + +return this; diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index cb9354b..8bd7968 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -1,3 +1,7 @@ +import hudson.plugins.git.* +import hudson.plugins.git.extensions.* +import hudson.plugins.git.extensions.impl.* + clh = load("${autobuilder.commonscripts()}/commandline/commandline.groovy") String getGitInfo(String workingSetRootPath) { @@ -6,7 +10,16 @@ String getGitInfo(String workingSetRootPath) { } } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true) { +@NonCPS +def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { + ArrayList sparsecheckoutpaths = new ArrayList() + for (path in paths) { + sparsecheckoutpaths.add(new SparseCheckoutPath(path)) + } + return new SparseCheckoutPaths(sparsecheckoutpaths) +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { reporootenv = env.REPO_ROOT if (reporootenv != null && repoRoot != "") { reporootpath = "${reporootenv.trim()}/${repoRoot}" @@ -28,7 +41,8 @@ def gitCheckout(String gitURL, String revision, String targetDir, String repoRoo [$class: 'CleanBeforeCheckout'], [$class: 'CloneOption', depth: cloneDepth, noTags: false, reference: reporootpath, shallow: shallowClone, timeout: 120], [$class: 'CheckoutOption', timeout: 120], - [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir] + [$class: 'RelativeTargetDirectory', relativeTargetDir: targetDir], + sparsecheckoutconfig(sparseitemlist) ], submoduleCfg: [], ]) From 6bf82480ff7719403ced72bbc9a4bb95db0d161b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 11:22:13 +0000 Subject: [PATCH 382/558] Only checkout repositories that have publishable items --- publish/do.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 2c5a018..c7084d0 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -104,11 +104,13 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { def String revision = "refs/tags/${releasemeta.buildstampinfo.buildstamp}" def publishableitems = releasemeta.publishableinfo def ArrayList publishablePaths = new ArrayList() - for (publishableitem in publishableitems) { - publishablePaths.add(publishableitem.filepath) - summaryappend("${publishableitem.filepath},") + if (publishableitems.size() > 0) { + for (publishableitem in publishableitems) { + publishablePaths.add(publishableitem.filepath) + summaryappend("${publishableitem.filepath},") + } + gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } - gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } def releasegitadd() { From 907c3668e81d0d6016616c4494aaec4318f801c2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:15:10 +0000 Subject: [PATCH 383/558] Build up folder structure for FTP --- publish/do.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index c7084d0..4d02b4f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -281,9 +281,9 @@ def publish(releasemeta, String uniquefolderforrepo) { } } -def recallcomponent(releaseinfo) { +def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -305,7 +305,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo) + recallcomponent(releaseinfo, customer, softwarerelease) } // publish(releasemeta, uniquefolderforrepo) From 7f98363f995c347a941916906f1f4e6521902b6f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 13:59:24 +0000 Subject: [PATCH 384/558] Change root path for transfer; Add an example FTP operation --- publish/do.groovy | 53 ++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 4d02b4f..b953f4c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -1,11 +1,6 @@ markunstable = false unstablereason = "" -FTP_CLIENT_LOCATION='ftp' -FTP_URL='ie1.hostedftp.com' -FTP_USER='ftp@globalinkjetsystems.com' -FTP_CUSTOMER_ROOT='/Customers/' -FTP_PASSWORD='ramshorn42' summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" @@ -250,40 +245,32 @@ def sourcegitpush(String dryrunpush ) { } } -def publish(releasemeta, String uniquefolderforrepo) { - summaryappend(" Git operations: ") - def String buildstamp = releasemeta.buildstampinfo.buildstamp +def String FTP_CLIENT_LOCATION='ftp' +def String FTP_URL='ie1.hostedftp.com' +def String FTP_USER='ftp@globalinkjetsystems.com' +def String FTP_CUSTOMER_ROOT='/Customers/' +def String FTP_PASSWORD='ramshorn42' - dir(uniquefolderforrepo) { - def String precommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Pre commit sha code: ${precommitshacode}" - - releasegitadd() +def publish() { + summaryappend(" FTP operations: ") - def boolean changed = checkforchange(buildmeta, previousreleasemeta) - if (changed) { - releasegitcommit(buildstamp) - } - else { - releasegitreset() - summaryappend("No-commit,") - } - - def String postcommitshacode = sh(script: "git rev-parse --verify HEAD", returnStdout: true).trim() - echo "Post commit sha code: ${postcommitshacode}" - - ArrayList taglist = maketaglist(buildstamp, otherreleasetags) - def tagadded = releasegittags(postcommitshacode, taglist) - - if (changed || tagadded) { - releasegitpush(dryrunpush) - } + dir(releaserepofolder) { + sh """#!/bin/bash + ftp -n -v ${FTP_URL} << EOT + ascii + user ${FTP_USER} ${FTP_PASSWORD} + prompt + cd ${FTP_CUSTOMER_ROOT} + ls -la + bye + EOT + """ } } def recallcomponent(releaseinfo, String customer, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/${PRODUCT_NAME}/${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -307,7 +294,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, customer, softwarerelease) } - // publish(releasemeta, uniquefolderforrepo) + publish() return [summarystring, markunstable, unstablereason] } From bc9dc68534aca9bd0480a5d1422b6ca2194de734 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 14:26:20 +0000 Subject: [PATCH 385/558] Build up folder structure for FTP --- publish/do.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index b953f4c..46b3fdd 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -245,15 +245,15 @@ def sourcegitpush(String dryrunpush ) { } } -def String FTP_CLIENT_LOCATION='ftp' -def String FTP_URL='ie1.hostedftp.com' -def String FTP_USER='ftp@globalinkjetsystems.com' -def String FTP_CUSTOMER_ROOT='/Customers/' -def String FTP_PASSWORD='ramshorn42' def publish() { summaryappend(" FTP operations: ") + def String FTP_CLIENT_LOCATION='ftp' + def String FTP_URL='ie1.hostedftp.com' + def String FTP_USER='ftp@globalinkjetsystems.com' + def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { sh """#!/bin/bash ftp -n -v ${FTP_URL} << EOT From 3b5e27f3e3f851a14e76506d0c893193e312af1e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:14:29 +0000 Subject: [PATCH 386/558] Use ncftpput to transfer --- publish/do.groovy | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 46b3fdd..13a280a 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -106,6 +106,9 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } gh.gitCheckout(componentURL, revision, repofolder, "", 1, true, publishablePaths) // Shallow clone } + else { + summaryappend("None ") + } } def releasegitadd() { @@ -245,32 +248,26 @@ def sourcegitpush(String dryrunpush ) { } } - -def publish() { +def publish(String customer) { summaryappend(" FTP operations: ") def String FTP_CLIENT_LOCATION='ftp' def String FTP_URL='ie1.hostedftp.com' def String FTP_USER='ftp@globalinkjetsystems.com' - def String FTP_CUSTOMER_ROOT='/Customers/' + def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { + summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") sh """#!/bin/bash - ftp -n -v ${FTP_URL} << EOT - ascii - user ${FTP_USER} ${FTP_PASSWORD} - prompt - cd ${FTP_CUSTOMER_ROOT} - ls -la - bye - EOT + ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ + summaryappend(", Transferred ") } } -def recallcomponent(releaseinfo, String customer, String softwarerelease) { +def recallcomponent(releaseinfo, String softwarerelease) { def releasemeta = releaseinfo.meta - def String uniquefolder = "${customer}/Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" + def String uniquefolder = "Software/${PRODUCT_NAME} ${software_release}/${releasemeta.productinfo.productname}_${releasemeta.productinfo.subcomponentname}" def String uniquefolderforrepo = "${releaserepofolder}/${uniquefolder}" def String uniquefoldertemp = "${tempfolder}/${uniquefolder}" @@ -292,9 +289,9 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin // For each component for (key in componentReleaseInfo.keySet()) { def releaseinfo = componentReleaseInfo[key] - recallcomponent(releaseinfo, customer, softwarerelease) + recallcomponent(releaseinfo, softwarerelease) } - publish() + publish(customer) return [summarystring, markunstable, unstablereason] } From 8229e666d8769f6b587c489899973c8b337a2c87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 15:59:04 +0000 Subject: [PATCH 387/558] Use ncftpput to transfer --- publish/do.groovy | 200 +++------------------------------------------- 1 file changed, 10 insertions(+), 190 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index 13a280a..8cdf9a3 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -5,12 +5,7 @@ summaryfolder = "${WORKSPACE}/summary" summaryfile = "${summaryfolder}/summary.txt" summarystring = "" releaserepofolder = "${WORKSPACE}/releaserepos" -sourcerepofolder = "${WORKSPACE}/sourcerepos" tempfolder = "${WORKSPACE}/temp" -gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" - -metascript = autobuilder.getmetafolderscript() -gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def resetfolders() { dir(summaryfolder) { @@ -42,54 +37,7 @@ def summaryappend(String summary) { summarystring += summary } -def getpreviousreleasemeta(buildmeta, String checkoutfolder) { - def previousreleasemeta = buildmeta.getNewMetaObject(this) - // See if there is meta data file from the current release - def String releasemetafolderpath = "${checkoutfolder}/${buildmeta.folder()}" - if (fileExists(releasemetafolderpath)) { - previousreleasemeta = metascript.metafromfolder(checkoutfolder) - } - return previousreleasemeta -} - -def checkoutreleaserepo(buildmeta, String checkoutfolder) { - def String releaserepo = buildmeta.releaseinfo.releaserepoURL - // Get the latest head revision of the release repo without checking out working copy; Check out to a folder unique to component; Extract any meta - def previousreleasemeta = buildmeta.getNewMetaObject(this) - dir(releaserepofolder) { - sh("git clone --depth 1 ${releaserepo} ${checkoutfolder}") - } - previousreleasemeta = getpreviousreleasemeta(buildmeta, checkoutfolder) - dir(checkoutfolder) { - // Clear all existing files - sh("git rm -r *") - } - return previousreleasemeta -} - -def checkoutsourcerepo(buildmeta, String checkoutfolder) { - def String sourcerepo = buildmeta.gitinfo.remoteurl - // Get the latest revision of the source repo without checking out working copy; Check out to a folder unique to component - if (fileExists(checkoutfolder)) { - dir(checkoutfolder) { - sh(""" - git reset --hard - git clean -fdx - git pull - # Remove all tags that are not already on the remote - git tag -l | xargs git tag -d >/dev/null - git fetch -t >/dev/null - """ - ) - } - } - else { - dir(sourcerepofolder) { - sh("git clone ${sourcerepo} ${checkoutfolder}") - } - } -} - +gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") @@ -111,143 +59,6 @@ def getcomponentartifacts(releaseinfo, String repofolder, String tempfolder) { } } -def releasegitadd() { - // Add anything new - sh("git add .") - summaryappend("Add,") -} - -def releasegitreset() { - // Add anything new - sh("git reset --hard") - sh("git clean -fdx") - summaryappend("Reset,") -} - -def releasegitcommit(String buildstamp) { - sh("git commit -a -m ${buildstamp}") - summaryappend("Commit,") -} - -// Determine if there has been a change by comparing the sha code in previous release meta and the build meta -def boolean checkforchange(buildmeta, previousreleasemeta) { - String buildsha = buildmeta.gitinfo.gitrevision - String previoussha = previousreleasemeta.gitinfo.gitrevision - - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" - return changed -} - -def ArrayList maketaglist(String buildstamp, String otherreleasetags) { - def ArrayList taglist = new ArrayList() - taglist.add(buildstamp) - - def ArrayList othertaglist = otherreleasetags.split(',') - for (othertagitem in othertaglist) { - if (othertagitem?.trim()) { - taglist.add(othertagitem.trim()) - } - } - return taglist -} - -def boolean releasegittag(String postcommitshacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != postcommitshacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The tag ${tag} has been used at another commit other than HEAD - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\"") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def boolean releasegittags(String postcommitshacode, ArrayList taglist) { - boolean tagadded = false - - summaryappend("Rel Tag ") - for (tag in taglist) { - if (releasegittag(postcommitshacode, tag)) { - tagadded = true - } - } - return tagadded -} - -def boolean sourcegittag(String shacode, String tag) { - def boolean tagadded = false - // Check if tag exists already - def int gitrevparsestatus = sh(script: "git rev-parse \"${tag}\" >\"${gitrevparseoutputfile}\" 2>/dev/null", returnStatus: true) - if (gitrevparsestatus == 0) { // Tag exists - def existingshacode = readFile(file: gitrevparseoutputfile).trim() - if (existingshacode != shacode) { // If the existing tag is not associated with HEAD - // This is a problem - The tag is already at another position in git repo - setunstable("The source tag ${tag} has been used at another commit (${existingshacode}) - You must decide what to do with it - Not good practice to delete it or move it") - summaryappend("(${tag} - Could not add!)") - } - else { - summaryappend("(${tag} - Already added)") - } - } - else { // If tag does not exist already - // Tag - sh("git tag \"${tag}\" ${shacode}") - summaryappend("(${tag})") - tagadded = true - } - return tagadded -} - -def sourcegittags(String buildsha, List taglist) { - boolean tagadded = false - - summaryappend(",Src Tag ") - for (tag in taglist) { - if (sourcegittag(buildsha, tag)) { - tagadded = true - } - } - return tagadded -} - -def releasegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags origin master") - summaryappend(",Dry-Run-Push") - setunstable("The artifact(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags origin master") - summaryappend(",Push") - } -} - -def sourcegitpush(String dryrunpush ) { - if (dryrunpush == "true") { - sh("git push -n --tags") - summaryappend(",Src-Dry-Run-Push") - setunstable("The source tag(s) have not been pushed/saved (\"Dry run push\" parameter/option is selected)") - } - else { - sh("git push --tags") - summaryappend(",Src Push") - } -} - def publish(String customer) { summaryappend(" FTP operations: ") @@ -278,6 +89,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { summaryappend("\n") } +def stripgithistory() { + dir(releaserepofolder) { + sh """#!/bin/bash + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + """ + } +} + def publishhierarchy(releasemeta, String customer, String softwarerelease, String maxdepth) { int md = maxdepth.toInteger() resetfolders() @@ -291,6 +110,7 @@ def publishhierarchy(releasemeta, String customer, String softwarerelease, Strin def releaseinfo = componentReleaseInfo[key] recallcomponent(releaseinfo, softwarerelease) } + stripgithistory() publish(customer) return [summarystring, markunstable, unstablereason] From 9a8a0b0c10505a9ae02007a0de8bc8cf6dd4ff10 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 16:01:13 +0000 Subject: [PATCH 388/558] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 8cdf9a3..34d0e42 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -68,7 +68,7 @@ def publish(String customer) { def String FTP_CUSTOMER_ROOT="/Customers/${customer}" def String FTP_PASSWORD='ramshorn42' dir(releaserepofolder) { - summaryappend(" FTP operations: Transferring to ${FTP_CUSTOMER_ROOT} ") + summaryappend("FTP operations: Transferring to ${FTP_CUSTOMER_ROOT}") sh """#!/bin/bash ncftpput -u ${FTP_USER} -p ${FTP_PASSWORD} -R ${FTP_URL} "${FTP_CUSTOMER_ROOT}" . """ From cf977356fc885ec7e011a437dbe8e7fdbdb7de58 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:12:45 +0000 Subject: [PATCH 389/558] Use ncftpput to transfer --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 34d0e42..a30bfc9 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -92,7 +92,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" ) | xargs rm -rf + ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf """ } } From 04cc212c89e0ae6f33c75d0e870fbdae5dc6785e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 18:53:08 +0000 Subject: [PATCH 390/558] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- artifacts/metaexport.groovy | 8 ++------ import/importer.groovy | 10 ++++++++-- versionnumbering/generate.groovy | 5 +++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index b4d4a6d..0ca5da7 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -4,7 +4,6 @@ def combineSubjobArtifacts(meta, subJobs) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") def ArrayList artifactFolders = new ArrayList() - int revisiontotal = 0 for (subJob in subJobs) { def subJobObject = subJob[0] if (subJobObject != null) { @@ -14,14 +13,11 @@ def combineSubjobArtifacts(meta, subJobs) { dir(destFolder) { deleteDir() } - importedmeta = importer.artifactimport(meta, subJobName, subJobNumber, destFolder) - revisiontotal += importedmeta.buildstampinfo.revision + importer.artifactimport(meta, subJobName, subJobNumber, destFolder) artifactFolders.add("${destFolder}/**") } } - def versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") - revisiontotal = revisiontotal % (versiongenerator.maxversionnumber() + 1) - return [artifactFolders, revisiontotal] + return artifactFolders } return this; diff --git a/import/importer.groovy b/import/importer.groovy index 4d24d30..081226a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,4 +1,6 @@ +versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") + def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', boolean fingerprint = false) { copyArtifacts( filter: filter, @@ -7,7 +9,9 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St selector: specific(buildnumber.toString()), target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', boolean fingerprint = false) { @@ -21,7 +25,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat ], target: destpath ) - return meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath) + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + return importedmeta } return this; diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index ae7e5f4..e73782f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,4 +136,9 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +int incrementrevision(int current, int increment) { + def int newrevision = (current + increment) % (maxversionnumber() + 1) + return newrevision +} + return this; \ No newline at end of file From bac7536940e8b6149c161038813c84b13798953e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 11 Dec 2018 21:25:38 +0000 Subject: [PATCH 391/558] Revision number needs to be altered if a job imports a dependent component; Make sure imports happen after the initial revision number is calculated --- versionnumbering/generate.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e73782f..7e5568a 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,6 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) + echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" return newrevision } From b73e5fd49fccefcd2b7922bce6cfd53ebe57edfb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:14:13 +0000 Subject: [PATCH 392/558] Remove .git files/folders before publishing --- publish/do.groovy | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index a30bfc9..91f3b3e 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,9 +90,14 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { + echo "Stripping git history out of ${releaserepo}" dir(releaserepofolder) { sh """#!/bin/bash - ( find . -type d -name ".git" && find . -name ".gitignore" && find . -name ".gitmodules" && find . -iname ".gitattributes") | xargs rm -rvf + pwd + find . -type d -name ".git" | xargs rm -rvf + find . -name ".gitignore" | xargs rm -rvf + find . -name ".gitmodules" | xargs rm -rvf + find . -iname ".gitattributes" | xargs rm -rvf """ } } From 517610eeab92a6d54cdb9ef1aa55a0afd1b9c602 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 11:29:40 +0000 Subject: [PATCH 393/558] Remove .git files/folders before publishing --- publish/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/do.groovy b/publish/do.groovy index 91f3b3e..ec9b53f 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -90,7 +90,7 @@ def recallcomponent(releaseinfo, String softwarerelease) { } def stripgithistory() { - echo "Stripping git history out of ${releaserepo}" + echo "Stripping git history out of ${releaserepofolder}" dir(releaserepofolder) { sh """#!/bin/bash pwd From f003a641ab95843999adc8ddcdc1020aa051b144 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 14:01:09 +0000 Subject: [PATCH 394/558] Releasing to git was relying on the source code sha changing only; However it needs to take into account that a dependent component revision number changed too --- release/do.groovy | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 169c292..85753e6 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -140,8 +140,18 @@ def boolean checkforchange(buildmeta, previousreleasemeta) { String buildsha = buildmeta.gitinfo.gitrevision String previoussha = previousreleasemeta.gitinfo.gitrevision - def boolean changed = (buildsha != previoussha) - echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${changed}" + def boolean shachanged = (buildsha != previoussha) + echo "buildsha: ${buildsha} previoussha: ${previoussha} changed: ${shachanged}" + + String buildstamp = buildmeta.buildstampinfo.buildstamp + String previousbuildstamp = previousreleasemeta.buildstampinfo.buildstamp + + def boolean buildstampchanged = (buildstamp != previousbuildstamp) + echo "buildstamp: ${buildstamp} previousbuildstamp: ${previousbuildstamp} changed: ${buildstampchanged}" + + def boolean changed = (shachanged | buildstampchanged) + echo "changed: ${changed}" + return changed } From 30ef4a61b2be74b7b610b107ce7dab05f89d2674 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:20:25 +0000 Subject: [PATCH 395/558] Remove .git detritus from published folders --- publish/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/publish/do.groovy b/publish/do.groovy index ec9b53f..e280c0c 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -94,10 +94,10 @@ def stripgithistory() { dir(releaserepofolder) { sh """#!/bin/bash pwd - find . -type d -name ".git" | xargs rm -rvf - find . -name ".gitignore" | xargs rm -rvf - find . -name ".gitmodules" | xargs rm -rvf - find . -iname ".gitattributes" | xargs rm -rvf + find . -type d -name ".git" | xargs -iXX rm -rvf XX + find . -name ".gitignore" | xargs -iXX rm -rvf XX + find . -name ".gitmodules" | xargs -iXX rm -rvf XX + find . -iname ".gitattributes" | xargs -iXX rm -rvf XX """ } } From 670bea0d2ccf2e2c38c348c7e4c6100923eaafa7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 12 Dec 2018 15:22:15 +0000 Subject: [PATCH 396/558] Move product agnostic scripts into common subtree area --- publish/publish.groovy | 90 ++++++++++++++++++++++++++++++++++++++++++ release/release.groovy | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 publish/publish.groovy create mode 100644 release/release.groovy diff --git a/publish/publish.groovy b/publish/publish.groovy new file mode 100644 index 0000000..8ed7aa8 --- /dev/null +++ b/publish/publish.groovy @@ -0,0 +1,90 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${SOFTWARE_RELEASE}" + SOURCEDIR = "${WORKSPACE}/whole" + SOURCEROOTDIR = 'whole' + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, SOURCEROOTDIR, 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + MAX_META_DEPTH='2' + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, CUSTOMER, SOFTWARE_RELEASE, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/release/release.groovy b/release/release.groovy new file mode 100644 index 0000000..828f1ff --- /dev/null +++ b/release/release.groovy @@ -0,0 +1,87 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.start(this, PRODUCT_NAME, SUBCOMPONENT_NAME, JIRA_KEY, RELEASE_REPO) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + filter: 'meta/**', + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Release components') { + steps { + script { + def releaser = load("${autobuilder.commonscripts()}/release/do.groovy") + (summarystring, markunstable, unstablereason) = releaser.releasehierarchy(buildmeta, MAX_META_DEPTH, OTHER_RELEASE_TAGS, DRY_RUN_PUSH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From d7c5d334283d9efc54c8547104431e911df39862 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 17:49:02 +0000 Subject: [PATCH 397/558] Remove redundant common_scripts --- file/utils.groovy | 16 --- versionnumbering/versioninfo.py | 180 -------------------------------- 2 files changed, 196 deletions(-) delete mode 100644 file/utils.groovy delete mode 100644 versionnumbering/versioninfo.py diff --git a/file/utils.groovy b/file/utils.groovy deleted file mode 100644 index 0ffa47c..0000000 --- a/file/utils.groovy +++ /dev/null @@ -1,16 +0,0 @@ - -def lineendings = load("${autobuilder.commonscripts()}/platform/lineendings.groovy") - -// Get current contents of given file and optionally add a newline string/character; If file doesn't exist, return default contents -def currentContents(file, newline = "${nl}", defaultContents = "") { - def contents = "" - if (fileExists("${file}")) { - contents = readFile(file: "${file}").trim() + newline - } - else { - contents = defaultContents - } - return contents -} - -return this; diff --git a/versionnumbering/versioninfo.py b/versionnumbering/versioninfo.py deleted file mode 100644 index 3d2a80d..0000000 --- a/versionnumbering/versioninfo.py +++ /dev/null @@ -1,180 +0,0 @@ -# encoding: utf-8 -''' -versionnumbering -- shortdesc - -versionnumbering generates a version folder with a common set of version information files in it - -@author: dan - -@copyright: 2017 Global Inkjet Systems Ltd. All rights reserved. -''' -import os -import sys -import shutil -import subprocess -import re -import datetime - -from argparse import ArgumentParser -from argparse import RawDescriptionHelpFormatter - -__all__ = [] -__version__ = 0.1 -__date__ = '2017-11-29' -__updated__ = '2017-11-29' - -# Handle: -# origin/master -# origin/2.0.x -def getVersionPrefix(sourcebranch): - print("Determining version number prefix from source branch name: {0}".format(sourcebranch)) - - defaultversionprefix = "1.30000.0." - mastertypeversionprefix = "1.20000.0." - thirdpartyversionprefix = "1.25000.0." - splits = sourcebranch.split('/') - if len(splits) < 1: - print("Source branch name is invalid. Defaulting to: {0}".format(defaultversionprefix)) - return defaultversionprefix - - sourcebranch = splits[-1] - - if sourcebranch.lower().find("master") != -1: # Special case - return mastertypeversionprefix - - splits2 = re.split(r'(\d+)', sourcebranch) - splits = [] - hasdot = False - for split in splits2: - if split.isdigit(): - splits.append(split) - if split == ".": - hasdot = True - - if not hasdot: - print("Source branch name has not dots in it (eg. 1.9.2). Defaulting to: {0}".format(thirdpartyversionprefix)) - return thirdpartyversionprefix - - versionprefix = "" - if len(splits) < 1: - print("This source branch name has no numbers in its name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - splitnumbercount = 0 - isIntermediateReleaseBranch = False - for split in splits: - if split.isdigit(): - versionprefix += split + "." - splitnumbercount += 1 - else: - if split == "x": - isIntermediateReleaseBranch = True - break - if splitnumbercount > 2: - break - if splitnumbercount == 0: - print("No leading numbers were found in source branch name so defaulting to: {0}".format(defaultversionprefix)) - versionprefix = defaultversionprefix - else: - if isIntermediateReleaseBranch: - versionprefix = mastertypeversionprefix - elif splitnumbercount == 1: - versionprefix += "0.0." - elif splitnumbercount == 2: - versionprefix += "0." - - print("Determined version prefix to be: {0}".format(versionprefix)) - return versionprefix - -def writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp): - with open(os.path.join(versionfolder, "prefix.txt"), 'w') as f: - f.write(versionprefix) - - with open(os.path.join(versionfolder, "revision.txt"), 'w') as f: - f.write(str(gitrevisioncount)) - - with open(os.path.join(versionfolder, "gitrevision.txt"), 'w') as f: - f.write(gitrevision) - - with open(os.path.join(versionfolder, "remoteinfo.txt"), 'w') as f: - f.write(remoteinfo) - - with open(os.path.join(versionfolder, "remoteurl.txt"), 'w') as f: - f.write(remoteurl) - - with open(os.path.join(versionfolder, "buildstamp.txt"), 'w') as f: - f.write(buildstamp) - - buildstampcommasep = buildstamp.replace(".", ",") - with open(os.path.join(versionfolder, "buildstampcommasep.txt"), 'w') as f: - f.write(buildstampcommasep) - - with open(os.path.join(versionfolder, "builddate.txt"), 'w') as f: - f.write(str(datetime.datetime.now())) - -def main(argv=None): # IGNORE:C0111 - '''Command line options.''' - - if argv is None: - argv = sys.argv - else: - sys.argv.extend(argv) - - program_name = os.path.basename(sys.argv[0]) - program_version = "v%s" % __version__ - program_build_date = str(__updated__) - program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date) - program_shortdesc = __import__('__main__').__doc__.split("\n")[1] - program_license = '''%s - - Created by dan on %s. - Copyright 2017 Global Inkjet Systems Ltd. All rights reserved. - - Distributed on an "AS IS" basis without warranties - or conditions of any kind, either express or implied. - -USAGE -''' % (program_shortdesc, str(__date__)) - - try: - # Setup argument parser - parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter) - parser.add_argument("-s", "--sourcebranch", dest="sourcebranch", action="store", default="origin/master", help="source branch name to discover version number [default: %(default)s]") - parser.add_argument("-v", "--versionfolder", dest="versionfolder", action="store", default="version", help="version folder to be created [default: %(default)s]") - parser.add_argument("-g", "--gitroot", dest="gitrootfolder", action="store", default="GitRepository", help="root path of source files [default: %(default)s]") - parser.add_argument("-b", "--baseversionnumber", dest="baseversionnumber", action="store", default=int(1000), help="base git version number [default: %(default)s]") - - # Process arguments - args = parser.parse_args() - - try: - sourcebranch = args.sourcebranch - versionfolder = args.versionfolder - gitrootfolder = args.gitrootfolder - baseversionnumber = args.baseversionnumber - - print("Source branch:", sourcebranch, "version folder:", versionfolder, "Git root:", gitrootfolder) - print("Base version number:", baseversionnumber) - - (gitrevisioncount, gitrevision, remoteinfo, remoteurl) = getGitRevision(gitrootfolder, baseversionnumber) - - versionprefix = getVersionPrefix(sourcebranch) - - buildstamp = versionprefix + str(gitrevisioncount) - print("Build stamp is: {0}".format(buildstamp)) - - writeversionfiles(versionfolder, versionprefix, gitrevisioncount, gitrevision, remoteinfo, remoteurl, buildstamp) - - return 0 - except Exception as e: - print("Error while creating version info:", e.__repr__()) - return 1 - - except Exception as e: - indent = len(program_name) * " " - print(program_name + ": " + repr(e) + "\n") - print(indent + " for help use --help") - return 2 - -if __name__ == "__main__": - sys.exit(main()) From 06c24df1afce0afef39cc46011715c3dc035493d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 15 Dec 2018 18:00:34 +0000 Subject: [PATCH 398/558] Remove compile retries --- compile/eclipsebuild/eclipseds5build.groovy | 52 ++++++++++----------- 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..cf382b1 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,39 +142,35 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ - } + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ } } - else { - echo "Not building ${build_variant} variant of project: ${project}" + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } - } catch (Exception e) { - echo "Build failed - Waiting for a while" - sleep(time: 4, unit: 'SECONDS') - error "Build failed..." } + else { + echo "Not building ${build_variant} variant of project: ${project}" + } + } catch (Exception e) { + error "Build failed..." } } } From e77d7391cfabb14f5cd0872ade82a52edf0c4106 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 16 Dec 2018 12:05:47 +0000 Subject: [PATCH 399/558] Revert "Remove compile retries" This reverts commit 90ee2b5c1c5c2bcf13af8d3c501114f41b3d8181. --- compile/eclipsebuild/eclipseds5build.groovy | 52 +++++++++++---------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index cf382b1..93a8064 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -142,35 +142,39 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou buildtype = "-build" } for (build_variant in variants) { - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' - echo End time: `date` - """ + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + """ + } } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" - """ + result = doExport(exportfolder, exportpathlist, "", build_variant) + if (result != 0) { + error "Failed to copy...." } } - result = doExport(exportfolder, exportpathlist, "", build_variant) - if (result != 0) { - error "Failed to copy...." + else { + echo "Not building ${build_variant} variant of project: ${project}" } + } catch (Exception e) { + echo "Build failed - Waiting for a while" + sleep(time: 4, unit: 'SECONDS') + error "Build failed..." } - else { - echo "Not building ${build_variant} variant of project: ${project}" - } - } catch (Exception e) { - error "Build failed..." } } } From c34da70d948cb5176022e5a0b202bcfe8ddc8cb9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:33:17 +0000 Subject: [PATCH 400/558] Basebuildnumber is now an external parameter --- versionnumbering/generate.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 7e5568a..f26806d 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,6 +136,11 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { + def int bbn = basebuildnumber.trim().toInteger() + return generate(meta, String sourcerevision, String sourcepath, bbn) +} + int incrementrevision(int current, int increment) { def int newrevision = (current + increment) % (maxversionnumber() + 1) echo "Current revision: ${current} Increment: ${increment} New revision: ${newrevision}" From 90a8c5a811f0992618fc92281f297c7450be766e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:46:09 +0000 Subject: [PATCH 401/558] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index f26806d..e2dc932 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -18,7 +18,7 @@ def getGitRevision(String gitrootpath, int basebuildnumber) { changecount = bat(returnStdout: true, script: "@git rev-list HEAD --count").trim().toInteger() } echo "Change count: ${changecount}" - adjustedchangecount = (changecount + basebuildnumber) % (maxversionnumber() + 1) + adjustedchangecount = ((changecount + basebuildnumber).abs()) % (maxversionnumber() + 1) echo "Adjusted change count: ${adjustedchangecount}" if (isUnix()) { gitrevision = sh(returnStdout: true, script: "git rev-parse HEAD").trim() From 5fcb3244fc4a6b2c55d96ba981b852658e11c882 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:50:06 +0000 Subject: [PATCH 402/558] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e2dc932..e28fa2f 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -138,7 +138,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { def int bbn = basebuildnumber.trim().toInteger() - return generate(meta, String sourcerevision, String sourcepath, bbn) + return generate(meta, sourcerevision, sourcepath, bbn) } int incrementrevision(int current, int increment) { From 423e1061f5bb69c70ca82e5a13ca6dfa2f135d7a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 19 Dec 2018 18:55:27 +0000 Subject: [PATCH 403/558] Basebuildnumber is now an external parameter - make sure it is positive --- versionnumbering/generate.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index e28fa2f..4b11cf0 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -136,7 +136,7 @@ String generate(meta, String sourcerevision, String sourcepath, int basebuildnum return buildstamp } -String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber = "1000") { +String generate(meta, String sourcerevision, String sourcepath, String basebuildnumber) { def int bbn = basebuildnumber.trim().toInteger() return generate(meta, sourcerevision, sourcepath, bbn) } From c8c0568c92ec17da36417ae26204158eeb647764 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 12 Jan 2019 12:35:30 +0000 Subject: [PATCH 404/558] On releasing, release the top level component last - so that we don't end up with partial release info --- meta/folder.groovy | 6 +++++- release/do.groovy | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 39de31f..ff56928 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -658,12 +658,16 @@ class MetaInfo extends GISJsonSerialised { return releasemetacollection } + String componentidentifier() { + return productinfo.productname + "_" + productinfo.subcomponentname + } + Map releasewalkerunique(int maxdepth, String startpath = "") { Map releasemetauniquecollection = new LinkedHashMap() ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) for (relatedmetainfo in releasemetacollection) { - String uniquestring = relatedmetainfo.meta.productinfo.productname + "_" + relatedmetainfo.meta.productinfo.subcomponentname + String uniquestring = relatedmetainfo.meta.componentidentifier() releasemetauniquecollection.put(uniquestring, relatedmetainfo) } return releasemetauniquecollection diff --git a/release/do.groovy b/release/do.groovy index 0ea3399..e264cc0 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -341,11 +341,18 @@ def releasehierarchy(buildmeta, String maxdepth, String otherreleasetags, String // Walk the meta information to a max depth, looking for artifacts to import for release componentReleaseInfo = buildmeta.releasewalkerunique(md) + def toplevelid = buildmeta.componentidentifier() // For each component for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] - releasecomponent(buildinfo, otherreleasetags, dryrunpush) + if (toplevelid != key) { + releasecomponent(buildinfo, otherreleasetags, dryrunpush) + } } + // Do the top level last + def toplevelbuildinfo = componentReleaseInfo[toplevelid] + releasecomponent(toplevelbuildinfo, otherreleasetags, dryrunpush) + return [summarystring, markunstable, unstablereason] } From 3274a8dd0999290c4a12e85432996d66d8520494 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 21 Jan 2019 15:15:27 +0000 Subject: [PATCH 405/558] Handle missing jq command --- differences/json.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/differences/json.groovy b/differences/json.groovy index 47d9cc5..19b7cf6 100644 --- a/differences/json.groovy +++ b/differences/json.groovy @@ -3,6 +3,8 @@ def comparefiles(importedjsonfile, currentjsonfile) { env.CURRENT_JSON="${currentjsonfile}" if (isUnix()) { sh """ + which jq + if [ \$? -ne 0 ]; then exit 1; fi # jq -S sorts json output diff -c <(jq -S . "${IMPORTED_JSON}") <(jq -S . "${CURRENT_JSON}") """ From 4df4b0d568ea3401acbef1bbacdd4f3bd31bf081 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 11:18:49 +0000 Subject: [PATCH 406/558] Certain components are built from the same source repository so therefore their version numbers should not be incremented when they are imported into a build component using the same source repository --- artifacts/metaexport.groovy | 5 +++-- import/importer.groovy | 12 ++++++++---- install/installhelper.groovy | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/artifacts/metaexport.groovy b/artifacts/metaexport.groovy index 0ca5da7..043c4de 100644 --- a/artifacts/metaexport.groovy +++ b/artifacts/metaexport.groovy @@ -9,11 +9,12 @@ def combineSubjobArtifacts(meta, subJobs) { if (subJobObject != null) { def subJobName = subJobObject.getProjectName() def subJobNumber = subJobObject.getNumber() - destFolder = subJob[1] + def destFolder = subJob[1] + def boolean incrementversionnumber = subJob[2] dir(destFolder) { deleteDir() } - importer.artifactimport(meta, subJobName, subJobNumber, destFolder) + importer.artifactimport(meta, subJobName, subJobNumber, destFolder, incrementversionnumber) artifactFolders.add("${destFolder}/**") } } diff --git a/import/importer.groovy b/import/importer.groovy index 1f89426..5c996b4 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -1,6 +1,6 @@ versiongenerator = load("${autobuilder.commonscripts()}/versionnumbering/generate.groovy") -def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -10,11 +10,13 @@ def artifactimport(meta, String jenkinsjob, int buildnumber, String destpath, St target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } -def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, String filter = '**', String excludes = '', boolean fingerprint = false) { +def artifactimport(meta, String jenkinsjob, String buildselector, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { copyArtifacts( filter: filter, excludes: excludes, @@ -27,7 +29,9 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat target: destpath ) def importedmeta = meta.importrelated(destpath) - meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } return importedmeta } diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 093ddef..03ee9e4 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -5,9 +5,9 @@ def Map artifactsImporter(meta, artifactDetails, installerDetails) { def importer = load("${autobuilder.commonscripts()}/import/importer.groovy") for (installer in toBuild) { def artifactKey = installerDetails.get(installer)[0] - (sourceProject, sourceselector, destFolder, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) + (sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy, binariesFolders) = artifactDetails.get(artifactKey) if (artifactFetched.get(artifactKey) == null) { - def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, toCopy, notToCopy) + def importmeta = importer.artifactimport(meta, sourceProject, sourceselector, destFolder, incrementversionnumber, toCopy, notToCopy) artifactFetched.put(artifactKey, importmeta) } } From bae55fa66609d54a7537bc71e9b4cbf2977dc9fd Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 22:14:31 +0000 Subject: [PATCH 407/558] Alter location of differences file so that it no longer gets published --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2ef8e22..461a8fb 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -419,6 +419,7 @@ class PublishableInfo extends GISJsonSerialised { class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" + private static String DIFFFOLDERNAME = "diff" private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" @@ -565,7 +566,7 @@ class MetaInfo extends GISJsonSerialised { } static String difffilepath() { - return METAFOLDERNAME + "/" + METADIFFFILENAME + return DIFFFOLDERNAME + "/" + METADIFFFILENAME } MetaInfo importrelated(String destpath) { From 553feb031b3a06fd2e2425e7cc023dee6885f249 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 28 Jan 2019 23:07:18 +0000 Subject: [PATCH 408/558] Alter location of differences file so that it no longer gets published --- meta/folder.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 461a8fb..efc8df2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -423,6 +423,7 @@ class MetaInfo extends GISJsonSerialised { private static String METAFILENAME = "info.json" private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" + private static String DIFFARTIFACTS = "${DIFFFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -621,7 +622,7 @@ class MetaInfo extends GISJsonSerialised { ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { - def String artifactlist = METAARTIFACTS + def String artifactlist = "${METAARTIFACTS},${DIFFARTIFACTS}" releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { artifactlist += "," + releaseitem From f37be2706ac69392bad0da925d25e1702cf80652 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 16:07:52 +0000 Subject: [PATCH 409/558] Try to create WORKSPACE relative paths --- meta/folder.groovy | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 52ceb06..245d961 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -554,13 +554,16 @@ class MetaInfo extends GISJsonSerialised { } String relativize(String path) { - def Path thispath = Paths.get(path) - if (thispath.startsWith(script.WORKSPACE)) { + def Path mypath = Paths.get(path) + String pathRelative + if (mypath.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(thispath) - return wsrelativePath.toString() + def Path wsrelativePath = wspath.relativize(mypath) + pathRelative = wsrelativePath.toString() + } else { + pathRelative = mypath.toString() } - return path + return pathRelative } static String metafilepath() { From bdcc0833e8dc003f022c9eaa7def5370403ea0de Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:05:25 +0000 Subject: [PATCH 410/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 245d961..d7cbc4f 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -556,12 +556,15 @@ class MetaInfo extends GISJsonSerialised { String relativize(String path) { def Path mypath = Paths.get(path) String pathRelative - if (mypath.startsWith(script.WORKSPACE)) { + script.echo("Seeing if ${path} begins with from ${script.WORKSPACE}") + if (path.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) pathRelative = wsrelativePath.toString() + script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}" } else { pathRelative = mypath.toString() + script.echo("Path relative is unchanged: ${pathRelative}" } return pathRelative } From 8bfaeda1e14ed816aeefebc6d07a595187dd1dfa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:07:33 +0000 Subject: [PATCH 411/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d7cbc4f..0c94a12 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -561,7 +561,7 @@ class MetaInfo extends GISJsonSerialised { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) pathRelative = wsrelativePath.toString() - script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}" + script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") } else { pathRelative = mypath.toString() script.echo("Path relative is unchanged: ${pathRelative}" From cd1226b83acc540ad3fb9754b3903b4970d42058 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:18:29 +0000 Subject: [PATCH 412/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 0c94a12..f6797a2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,7 +564,7 @@ class MetaInfo extends GISJsonSerialised { script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") } else { pathRelative = mypath.toString() - script.echo("Path relative is unchanged: ${pathRelative}" + script.echo("Path relative is unchanged: ${pathRelative}") } return pathRelative } From 3926df768bf61fdd3ef92980598c5f8fc65eb2d8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:21:14 +0000 Subject: [PATCH 413/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index f6797a2..ce041d0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -556,7 +556,7 @@ class MetaInfo extends GISJsonSerialised { String relativize(String path) { def Path mypath = Paths.get(path) String pathRelative - script.echo("Seeing if ${path} begins with from ${script.WORKSPACE}") + script.echo("Seeing if ${path} begins with ${script.WORKSPACE}") if (path.startsWith(script.WORKSPACE)) { def Path wspath = Paths.get(script.WORKSPACE) def Path wsrelativePath = wspath.relativize(mypath) From eb37e310949a8bbceb71dac47c5285a89b9b1b6d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 17:45:39 +0000 Subject: [PATCH 414/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index ce041d0..74c00cc 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -553,17 +553,14 @@ class MetaInfo extends GISJsonSerialised { deserialiseJson(rawmeta) } - String relativize(String path) { - def Path mypath = Paths.get(path) + String relativize(String mypath) { String pathRelative - script.echo("Seeing if ${path} begins with ${script.WORKSPACE}") - if (path.startsWith(script.WORKSPACE)) { - def Path wspath = Paths.get(script.WORKSPACE) - def Path wsrelativePath = wspath.relativize(mypath) - pathRelative = wsrelativePath.toString() - script.echo("Extracted ${pathRelative} from ${wspath.toString()} and ${mypath.toString()}") + script.echo("Seeing if ${mypath} begins with ${script.WORKSPACE}") + if (mypath.startsWith(script.WORKSPACE)) { + pathRelative = mypath.minus(script.WORKSPACE) + script.echo("Extracted ${pathRelative} from ${script.WORKSPACE} and ${mypath}") } else { - pathRelative = mypath.toString() + pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } return pathRelative From c8fa1243fd3c38062517254701a26c7bf11414d7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Feb 2019 18:07:16 +0000 Subject: [PATCH 415/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 74c00cc..1f90c61 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -563,6 +563,13 @@ class MetaInfo extends GISJsonSerialised { pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } + // Strip leading / if any + pathRelative = pathRelative.replaceAll("^/+", "") + // Strip leading \ if any + pathRelative = pathRelative.replaceAll("^\\+", "") + + script.echo("Path relative is ${pathRelative}") + return pathRelative } From 4425de19dcbca4b1e9f8843040f7c0a4052bae84 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:06:38 +0000 Subject: [PATCH 416/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 1f90c61..3a4d464 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -564,9 +564,11 @@ class MetaInfo extends GISJsonSerialised { script.echo("Path relative is unchanged: ${pathRelative}") } // Strip leading / if any - pathRelative = pathRelative.replaceAll("^/+", "") + pathRelative = pathRelative.replaceAll('^/+', '') // Strip leading \ if any - pathRelative = pathRelative.replaceAll("^\\+", "") + pathRelative = pathRelative.replaceAll('^\\+', '') + // Convert \ with / + pathRelative = pathRelative.replaceAll('\\', '/') script.echo("Path relative is ${pathRelative}") From d7ecd0f1809de688e109ffefdddc8cc1dd7069cb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:23:02 +0000 Subject: [PATCH 417/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 3a4d464..adfee97 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -568,7 +568,7 @@ class MetaInfo extends GISJsonSerialised { // Strip leading \ if any pathRelative = pathRelative.replaceAll('^\\+', '') // Convert \ with / - pathRelative = pathRelative.replaceAll('\\', '/') + pathRelative = pathRelative.replace('\\', '/') script.echo("Path relative is ${pathRelative}") From dedcdfd5e964d005d59ec7cffa50b0ae7bf3715d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Feb 2019 08:46:23 +0000 Subject: [PATCH 418/558] offsetpath field in meta files was absolute; should be relative to workspace --- meta/folder.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index adfee97..f9d4bf3 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -563,12 +563,10 @@ class MetaInfo extends GISJsonSerialised { pathRelative = mypath script.echo("Path relative is unchanged: ${pathRelative}") } + // Convert \ with / + pathRelative = pathRelative.tokenize('\\').join('/') // Strip leading / if any pathRelative = pathRelative.replaceAll('^/+', '') - // Strip leading \ if any - pathRelative = pathRelative.replaceAll('^\\+', '') - // Convert \ with / - pathRelative = pathRelative.replace('\\', '/') script.echo("Path relative is ${pathRelative}") From c7d58a0e48e912ac5156daef3b74544e223675a4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 00:01:14 +0000 Subject: [PATCH 419/558] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 42 ++++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 93a8064..d333af3 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -28,22 +28,50 @@ def doKillLeftovers() { } } +def doCheck(checkpathlist, build_variant = "") { + def result = 0 + for (checkpath in checkpathlist) { + echo "Checking file at: ${checkpath}" + script { + echo "Build variant is ${build_variant}" + if (build_variant) { + def srcFile = "${WORKSPACE}/${checkpath}" + def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + result = sh( script: """ + // Print out dependencies of the built file + readelf -d "${srcFileExpanded}" | grep NEEDED + if [ \$? -ne 0 ]; then exit 1; fi + """, + returnStatus: true + ) + if (result != 0) { + return result + } + } + } + } + return result +} + def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { def result = 0 + def destFolder = "" + def srcFile = "" + def srcFileExpanded = "" for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - env.destFolder = "${exportfolder}/${build_variant}" - env.srcFile = "${WORKSPACE}/${exportpath}" - env.srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + destFolder = "${exportfolder}/${build_variant}" + srcFile = "${WORKSPACE}/${exportpath}" + srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" - env.destFolder = "${exportfolder}/${project}" - env.srcFileExpanded = "${WORKSPACE}/${exportpath}" + destFolder = "${exportfolder}/${project}" + srcFileExpanded = "${WORKSPACE}/${exportpath}" } } dir (env.destFolder) { @@ -162,6 +190,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou """ } } + result = doCheck(exportpathlist, build_variant) + if (result != 0) { + error "Failed to check files...." + } result = doExport(exportfolder, exportpathlist, "", build_variant) if (result != 0) { error "Failed to copy...." From 1f771c6290ba633f262935293a0b59cb913ddc52 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 00:33:36 +0000 Subject: [PATCH 420/558] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index d333af3..2dbd6ea 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -74,7 +74,7 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { srcFileExpanded = "${WORKSPACE}/${exportpath}" } } - dir (env.destFolder) { + dir (destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" if (isUnix()) { result = sh( script: """ @@ -88,10 +88,10 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } } else { - result = bat( script: ''' - copy \"%srcFileExpanded%\" \"%destFolder%\" + result = bat( script: """ + copy "${srcFileExpanded}" "${destFolder}" if errorlevel 1 EXIT /b 1 - ''', + """, returnStatus: true ) if (result != 0) { From 2ace18a3793095da848291cab4eae63e00ea20c6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Feb 2019 07:55:15 +0000 Subject: [PATCH 421/558] Do a dependency check on each binary produced --- compile/eclipsebuild/eclipseds5build.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 2dbd6ea..302b4f8 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -37,8 +37,9 @@ def doCheck(checkpathlist, build_variant = "") { if (build_variant) { def srcFile = "${WORKSPACE}/${checkpath}" def srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") + // Print out dependencies of the built file + echo "Showing dependencies of ${srcFileExpanded}" result = sh( script: """ - // Print out dependencies of the built file readelf -d "${srcFileExpanded}" | grep NEEDED if [ \$? -ne 0 ]; then exit 1; fi """, From ec6d8e9e0eb1b740c077bb5a6b20cba03159ce45 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:37:17 +0000 Subject: [PATCH 422/558] Add some checking job scripts --- check/build_meta.groovy | 86 +++++++++++++++++++++++++++++++++++++++++ check/do_meta.groovy | 64 ++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 check/build_meta.groovy create mode 100644 check/do_meta.groovy diff --git a/check/build_meta.groovy b/check/build_meta.groovy new file mode 100644 index 0000000..34eed98 --- /dev/null +++ b/check/build_meta.groovy @@ -0,0 +1,86 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + autobuilderscript = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy") + meta = autobuilderscript.startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/build_artifacts", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Get build artifact meta') { + steps { + script { + BUILD_ARTIFACTS = "${WORKSPACE}/build_artifacts" + copyArtifacts( + projectName: BUILD_JOB, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: BUILD_JOB_SELECTOR + ], + target: BUILD_ARTIFACTS + ) + def mfs = autobuilderscript.getmetafolderscript() + buildmeta = mfs.metafromfolder(BUILD_ARTIFACTS) + } + } + } + stage('Check components') { + steps { + script { + def checker = load("${autobuilder.commonscripts()}/check/do_meta.groovy") + (summarystring, markunstable, unstablereason) = checker.checkhierarchy(buildmeta, MAX_META_DEPTH, BUILD_ARTIFACTS) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/check/do_meta.groovy b/check/do_meta.groovy new file mode 100644 index 0000000..75353da --- /dev/null +++ b/check/do_meta.groovy @@ -0,0 +1,64 @@ +markunstable = false +unstablereason = "" + +summaryfolder = "${WORKSPACE}/summary" +summaryfile = "${summaryfolder}/summary.txt" +summarystring = "" +releaserepofolder = "${WORKSPACE}/releaserepos" +sourcerepofolder = "${WORKSPACE}/sourcerepos" +tempfolder = "${WORKSPACE}/temp" +gitrevparseoutputfile = "${tempfolder}/gitrevparse.out" + +metascript = autobuilder.getmetafolderscript() + +def resetfolders() { + dir(summaryfolder) { + deleteDir() + } + dir(releaserepofolder) { + deleteDir() + } + dir(tempfolder) { + deleteDir() + } +} + +def preconditions() { + markunstable = false + unstablereason = "" + summarystring = "" + writeFile(file: summaryfile, text: "Release summary: \n") +} + +def setunstable(String reason) { + markunstable = true + unstablereason += reason + "\n" + echo "Marked build as unstable: Reason: ${reason}" +} + +def summaryappend(String summary) { + def String currentsummarycontents = readFile(file: summaryfile) + writeFile(file: summaryfile, text: currentsummarycontents + summary) + summarystring += summary +} + +def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { + int md = maxdepth.toInteger() + resetfolders() + preconditions() + + // Walk the meta information to a max depth, looking for components to release + def componentReleaseInfo = buildmeta.releasewalkerunique(md) + echo "Unique components:" + def String versioncomponent = "" + for (key in componentReleaseInfo.keySet()) { + def buildinfo = componentReleaseInfo[key] + def offsetpath = buildinfo.offsetpath + versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + } + echo versioncomponent + + return [summarystring, markunstable, unstablereason] +} + +return this; From f68b9241823cc105c7affccac31a97974aa481d5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 10:44:08 +0000 Subject: [PATCH 423/558] Add some checking job scripts --- check/do_meta.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 75353da..6712a11 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,14 +47,15 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() - // Walk the meta information to a max depth, looking for components to release + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath - versioncomponent += "Key: ${key} Path: ${offsetpath}\n" + def abspath = "${buildpathroot}/${offsetpath}" + versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 719e82d40bea95e56e979389285456611e3dbeb0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:07:24 +0000 Subject: [PATCH 424/558] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 6712a11..e01aa41 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key} Path: ${offsetpath} File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 2cad1ff24eca3e72fc1f0c69f8c41f400f2cd88f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:12:31 +0000 Subject: [PATCH 425/558] Add some checking job scripts --- check/do_meta.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index e01aa41..7fce1eb 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -55,7 +55,7 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { def buildinfo = componentReleaseInfo[key] def offsetpath = buildinfo.offsetpath def abspath = "${buildpathroot}/${offsetpath}" - versioncomponent += "Key: ${key}Path: ${offsetpath}File exists in import: ${fileExists(abspath)}\n" + versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } echo versioncomponent From 36ec063feea07fc2c17f4007f327400ab9485286 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:22:58 +0000 Subject: [PATCH 426/558] Add some checking job scripts --- check/do_meta.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 7fce1eb..cb19ddf 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -47,6 +47,18 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { resetfolders() preconditions() + // Walk the meta information to a max depth + def buildinfo = buildmeta.releasewalker(md) + echo "All components:" + def String componentinfo = "" + for (entry in buildinfo) { + def name = buildinfo.meta.componentversionuniqueidentifier() + def offsetpath = buildinfo.offsetpath + def abspath = "${buildpathroot}/${offsetpath}" + componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" + } + echo componentinfo + // Walk the meta information to a max depth def componentReleaseInfo = buildmeta.releasewalkerunique(md) echo "Unique components:" From 1ad8cbb7366ddfac2433ed62a38b5660486e27eb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:25:25 +0000 Subject: [PATCH 427/558] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index cb19ddf..8935ca5 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -64,8 +64,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "Unique components:" def String versioncomponent = "" for (key in componentReleaseInfo.keySet()) { - def buildinfo = componentReleaseInfo[key] - def offsetpath = buildinfo.offsetpath + def buildinfo2 = componentReleaseInfo[key] + def offsetpath = buildinfo2.offsetpath def abspath = "${buildpathroot}/${offsetpath}" versioncomponent += "Key: ${key} \t\t\tPath: ${offsetpath} \t\t\tFile exists in import: ${fileExists(abspath)}\n" } From ed9284264c32a817e513514927c4834db43af639 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:29:23 +0000 Subject: [PATCH 428/558] Add some checking job scripts --- check/do_meta.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/check/do_meta.groovy b/check/do_meta.groovy index 8935ca5..7bfe2d1 100644 --- a/check/do_meta.groovy +++ b/check/do_meta.groovy @@ -52,8 +52,8 @@ def checkhierarchy(buildmeta, String maxdepth, String buildpathroot) { echo "All components:" def String componentinfo = "" for (entry in buildinfo) { - def name = buildinfo.meta.componentversionuniqueidentifier() - def offsetpath = buildinfo.offsetpath + def name = entry.meta.componentversionuniqueidentifier() + def offsetpath = entry.offsetpath def abspath = "${buildpathroot}/${offsetpath}" componentinfo += "Name: ${name} \t\t\t\tPath: ${offsetpath} \t\t\t\tFile exists in import: ${fileExists(abspath)}\n" } From d90b8a2b3725454941514cdce985b70217b88473 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:51:03 +0000 Subject: [PATCH 429/558] Add some checking job scripts --- meta/folder.groovy | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index d751f5e..a18e8d7 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,6 +687,18 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } + Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + if (releasemetacollection.containsKey(uniquestring)) { + def existingentry = releasemetacollection[uniquestring] + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + } + } else { + releasemetauniquecollection.put(uniquestring, relatedmetainfo) + } + return releasemetacollection + } + Map releasewalkerunique(int maxdepth, String startpath = "") { def ArrayList releasemetacollection = new ArrayList() releasemetacollection = releasewalkerp(maxdepth, releasemetacollection, startpath) @@ -694,7 +706,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetauniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentidentifier() - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetauniquecollection = putwithshortestoffsetpath(releasemetauniquecollection, uniquestring, relatedmetainfo) } return releasemetauniquecollection @@ -707,7 +719,7 @@ class MetaInfo extends GISJsonSerialised { def Map releasemetaversionuniquecollection = new LinkedHashMap() for (relatedmetainfo in releasemetacollection) { def String uniquestring = relatedmetainfo.meta.componentversionuniqueidentifier() - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetaversionuniquecollection = putwithshortestoffsetpath(releasemetaversionuniquecollection, uniquestring, relatedmetainfo) } return releasemetaversionuniquecollection From d226eba99021f0811526b01518b286dd336d40fa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:54:50 +0000 Subject: [PATCH 430/558] Add some checking job scripts --- meta/folder.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a18e8d7..a8ffa41 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -687,14 +687,14 @@ class MetaInfo extends GISJsonSerialised { return productinfo.productname + "_" + productinfo.subcomponentname + "_" + buildstampinfo.buildstamp } - Map putwithshortestoffsetpath(ArrayList releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { - releasemetaversionuniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } } else { - releasemetauniquecollection.put(uniquestring, relatedmetainfo) + releasemetacollection.put(uniquestring, relatedmetainfo) } return releasemetacollection } From 22cd92580cabbf422c2bf7dd44ac2924751e9206 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 11:57:14 +0000 Subject: [PATCH 431/558] Add some checking job scripts --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index a8ffa41..1bf855c 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -690,7 +690,7 @@ class MetaInfo extends GISJsonSerialised { Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] - if (existingentry.offsetpath.size() > relatedmetainfo.offsetpathsize()) { + if (existingentry.offsetpath.size() > relatedmetainfo.offsetpath.size()) { releasemetacollection.put(uniquestring, relatedmetainfo) } } else { From e1bc45fafb9df2ef8d2048fc4b195de9dce6fa36 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 24 Mar 2019 12:21:15 +0000 Subject: [PATCH 432/558] Create a script to copy logic to a build job - so that it will be stamped during release --- copy/logic.groovy | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 copy/logic.groovy diff --git a/copy/logic.groovy b/copy/logic.groovy new file mode 100644 index 0000000..6f8f633 --- /dev/null +++ b/copy/logic.groovy @@ -0,0 +1,56 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean exports') { + steps { + script { + cleanfolders = [ + "${WORKSPACE}/logic", + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying logic artifacts') { + steps { + script { + env.LOGICDEST = "${WORKSPACE}/logic" + if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["logic/**"]) + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} From 55f8109e5fc2c87e8f9acae6be2741948b8ab3bc Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Mar 2019 14:21:19 +0000 Subject: [PATCH 433/558] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6f8f633..d951ffa 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -35,6 +35,9 @@ pipeline { script { def artifactlist = meta.generateartifacts(["logic/**"]) archiveArtifacts "${artifactlist}" + sh """ + cp -v -R logic/meta/info.json meta/info.json + """ } } } From 16b2bd8acd31d599a7aa6983199c23c6c2a635be Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 26 Mar 2019 14:25:25 +0000 Subject: [PATCH 434/558] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index d951ffa..845f676 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -34,10 +34,10 @@ pipeline { steps { script { def artifactlist = meta.generateartifacts(["logic/**"]) - archiveArtifacts "${artifactlist}" sh """ cp -v -R logic/meta/info.json meta/info.json """ + archiveArtifacts "${artifactlist}" } } } From aeb2669473c83d694e89cdb3a0d6b197eed2a1e9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 08:56:29 +0000 Subject: [PATCH 435/558] Handle the artifact source parameter --- import/importer.groovy | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/import/importer.groovy b/import/importer.groovy index 5c996b4..40151e1 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -35,4 +35,18 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat return importedmeta } +def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + switch (source) { + case "Jenkins job artifact": { + return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + case "Release repository": { + return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) + } + default: + error("Unknown type of artifact source: ${source}") + } + return meta +} + return this; From a6ce45b21441f9f79aa4ed0fa14b87f3bac536b2 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 08:59:37 +0000 Subject: [PATCH 436/558] Handle the artifact source parameter --- import/importer.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 40151e1..abf3b2f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -37,12 +37,10 @@ def artifactimport(meta, String jenkinsjob, String buildselector, String destpat def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { - case "Jenkins job artifact": { + case "Jenkins job artifact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) - } - case "Release repository": { + case "Release repository": return artifactimportfromrepo(meta, repo, revision, destpath, incrementversionnumber, filter, excludes, fingerprint) - } default: error("Unknown type of artifact source: ${source}") } From 56d3088733d632579ad4b5ebca9e2439d0de1ab1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 09:16:18 +0000 Subject: [PATCH 437/558] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 845f676..fae69b4 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -13,6 +13,7 @@ pipeline { script { cleanfolders = [ "${WORKSPACE}/logic", + "${WORKSPACE}/embedded_logic", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -33,9 +34,10 @@ pipeline { stage('Archiving artifacts') { steps { script { - def artifactlist = meta.generateartifacts(["logic/**"]) + def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - cp -v -R logic/meta/info.json meta/info.json + rm -v meta/info.json + mv -v logic/embedded_logic . """ archiveArtifacts "${artifactlist}" } From d4812874ab9c1455806a58eb438b7f5ec7e9d33a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 27 Mar 2019 09:19:44 +0000 Subject: [PATCH 438/558] Overwrite the benign copy meta with the logic meta --- copy/logic.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index fae69b4..6e97510 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,8 +36,8 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v meta/info.json - mv -v logic/embedded_logic . + rm -v -rf meta + mv -v logic/* . """ archiveArtifacts "${artifactlist}" } From 4942d1bc6bbf30c5a16c23cf5070d15ec0da0345 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:44:12 +0000 Subject: [PATCH 439/558] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ba4d96b..a474b6e 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -29,15 +29,50 @@ def guessreporoot(componentURL) { return reporoot } -def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { - reporootenv = env.REPO_ROOT - if (reporootenv != null && repoRootSuffix != "") { - reporootpath = "${reporootenv.trim()}/${repoRootSuffix}" - echo "Using release repo: ${reporootpath}" - } else { - reporootpath = "" +def String uniquereporootname(String gitURL) { + def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") + echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" + return uniquename +} + +// The REPO_ROOT is a folder where a cache of git checkouts can be kept - each subfolder name is unique to a particular git repo +def String checkouttoreporoot(String gitURL) { + def String reporootsubfolder + def String reporootenv = env.REPO_ROOT + if (!reporootenv) { + reporootsubfolder = "" echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" - } + } else { + def reporootname = uniquereporootname(gitURL) + def reporootsubfolder = "${reporootenv}/${reporootname}" + + if (!fileExists(reporootsubfolder)) { + echo "Checking out ${gitURL} to ${reporootname}" + checkout([ + $class: 'GitSCM', + userRemoteConfigs: [ + [url: gitURL] + ], + branches: [ + [name: 'origin/master'] + ], + doGenerateSubmoduleConfigurations: false, + extensions: [ + [$class: 'GitLFSPull'], + [$class: 'CleanBeforeCheckout'], + [$class: 'CheckoutOption', timeout: 180], + [$class: 'RelativeTargetDirectory', relativeTargetDir: reporootsubfolder], + ], + submoduleCfg: [], + ]) + } else { + echo "Found ${gitURL} checked out at ${reporootname}" + } + return reporootsubfolder +} + +def gitCheckout(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { + def reporootpath = checkouttoreporoot(gitURL) checkout([ $class: 'GitSCM', userRemoteConfigs: [ From d3e547b6c447942fcdcdb043a4786cd1f5cda515 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:48:35 +0000 Subject: [PATCH 440/558] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index a474b6e..bd1b15e 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -19,16 +19,6 @@ def SparseCheckoutPaths sparsecheckoutconfig(ArrayList paths) { return new SparseCheckoutPaths(sparsecheckoutpaths) } -def guessreporoot(componentURL) { - if (!env.REPO_ROOT || env.REPO_ROOT.size() == 0) { - return '' - } - def guess = componentURL.replaceFirst(/.*\/(.*).git/, '$1') - def reporoot = "${REPO_ROOT}/${guess}" - echo "Guessing reporoot is: ${reporoot}" - return reporoot -} - def String uniquereporootname(String gitURL) { def uniquename = gitURL.replaceAll("[^A-Za-z0-9]", "_") echo "Unique reporoot name for URL: ${gitURL} is: ${uniquename}" @@ -68,6 +58,7 @@ def String checkouttoreporoot(String gitURL) { } else { echo "Found ${gitURL} checked out at ${reporootname}" } + } return reporootsubfolder } From b598a99b2e675b41dccf1a98173dcd4d1ffdfd6c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:49:54 +0000 Subject: [PATCH 441/558] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index bd1b15e..ea3501c 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,7 +34,7 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - def reporootsubfolder = "${reporootenv}/${reporootname}" + reporootsubfolder = "${reporootenv}/${reporootname}" if (!fileExists(reporootsubfolder)) { echo "Checking out ${gitURL} to ${reporootname}" From 2a8f6327ecf68ef03b27e5c852a6ceb924b26303 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:52:55 +0000 Subject: [PATCH 442/558] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index ea3501c..432316f 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,10 +34,10 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootname}" + reporootsubfolder = "${reporootenv}/${reporootsubfolder}" if (!fileExists(reporootsubfolder)) { - echo "Checking out ${gitURL} to ${reporootname}" + echo "Checking out ${gitURL} to ${reporootsubfolder}" checkout([ $class: 'GitSCM', userRemoteConfigs: [ @@ -56,7 +56,7 @@ def String checkouttoreporoot(String gitURL) { submoduleCfg: [], ]) } else { - echo "Found ${gitURL} checked out at ${reporootname}" + echo "Found ${gitURL} checked out at ${reporootsubfolder}" } } return reporootsubfolder From 6beb745a69a25cd13757b1bbcbe09cff04aed2c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 29 Mar 2019 08:56:01 +0000 Subject: [PATCH 443/558] Always create a repo_root copy of a repo when necessary - in order to speed up jobs --- versioncontrol/git.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 432316f..5599dc6 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -34,7 +34,7 @@ def String checkouttoreporoot(String gitURL) { echo "To improve Git performance: Define the REPO_ROOT enviroment to a path where a cache of git repositories can be placed on this slave" } else { def reporootname = uniquereporootname(gitURL) - reporootsubfolder = "${reporootenv}/${reporootsubfolder}" + reporootsubfolder = "${reporootenv}/${reporootname}" if (!fileExists(reporootsubfolder)) { echo "Checking out ${gitURL} to ${reporootsubfolder}" From ad00620c74ff87efb91ed4c53d309123d2037e97 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 15:53:44 +0100 Subject: [PATCH 444/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 45 +++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 819a580..d064161 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -30,4 +30,47 @@ def placeBinaries(artifactDetails, installerDetails) { } } -return this; \ No newline at end of file +def checkISMfile(String ISMfile) { + def xmlFile = new File(ISMfile) + + def parsedxml = new XmlParser().parse(xmlFile) + def pathvariableinfo = [:] + def exitcode = 0 + // Load up a map of path variable and path pairs + // Ignore blank path definitions + // Ignore the workspace definition which should have the only absolute path + parsedxml.each{ table -> + if (table.@name == "ISPathVariable") { + table.row.each{ row -> + def pathname = row.td[0].value()[0] + def path = row.td[1].value()[0] + if (path && pathname != "WORKSPACE") { + pathvariableinfo[pathname] = path + } + } + } + } + // Check that each path variable is defined beginning with a path variable or the WORKSPACE one + pathvariableinfo.each{ pair -> + println "pair key: ${pair.key} value: ${pair.value}" + def path = pair.value + def pathvarmatcher = path =~ /<(.*)>.*/ + if (pathvarmatcher.matches()) { + def pathvarfound = pathvarmatcher.group(1) + if (pathvarfound != 'WORKSPACE') { + if (!pathvariableinfo.containsKey(pathvarfound)) { + println "Undefined path variable ${pathvarfound} found" + exitcode = 1 + } + } + } else { + println "Path: ${path} does not begin with a path variable" + exitcode = 1 + } + } + if (exitcode > 0) { + error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" + } +} + +return this; From 1435d338acc9019903594c3b2298bd4ab3150f2b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 17:58:05 +0100 Subject: [PATCH 445/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index d064161..886c604 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -31,9 +31,9 @@ def placeBinaries(artifactDetails, installerDetails) { } def checkISMfile(String ISMfile) { - def xmlFile = new File(ISMfile) + def xml = readFile(ISMfile) - def parsedxml = new XmlParser().parse(xmlFile) + def parsedxml = new XmlParser().parseText(xml) def pathvariableinfo = [:] def exitcode = 0 // Load up a map of path variable and path pairs From ee7da8715fc2b4f4b8f2fb269207746d83d409ca Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:07:31 +0100 Subject: [PATCH 446/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 886c604..6cecb85 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -33,7 +33,7 @@ def placeBinaries(artifactDetails, installerDetails) { def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) - def parsedxml = new XmlParser().parseText(xml) + def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] def exitcode = 0 // Load up a map of path variable and path pairs From 63202270913c56490108966727ca6edca7e11950 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:21:53 +0100 Subject: [PATCH 447/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6cecb85..f3dcdd4 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -30,6 +30,7 @@ def placeBinaries(artifactDetails, installerDetails) { } } +@NonCPS def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) From 1c10bfba373c73007a0ca0cae5cb2ec611bd0f24 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 8 Apr 2019 18:30:16 +0100 Subject: [PATCH 448/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index f3dcdd4..823c117 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -33,7 +33,8 @@ def placeBinaries(artifactDetails, installerDetails) { @NonCPS def checkISMfile(String ISMfile) { def xml = readFile(ISMfile) - + echo "Checking ISM file: ${ISMfile}" + def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] def exitcode = 0 @@ -53,24 +54,26 @@ def checkISMfile(String ISMfile) { } // Check that each path variable is defined beginning with a path variable or the WORKSPACE one pathvariableinfo.each{ pair -> - println "pair key: ${pair.key} value: ${pair.value}" + echo "pair key: ${pair.key} value: ${pair.value}" def path = pair.value def pathvarmatcher = path =~ /<(.*)>.*/ if (pathvarmatcher.matches()) { def pathvarfound = pathvarmatcher.group(1) if (pathvarfound != 'WORKSPACE') { if (!pathvariableinfo.containsKey(pathvarfound)) { - println "Undefined path variable ${pathvarfound} found" + echo "Undefined path variable ${pathvarfound} found" exitcode = 1 } } } else { - println "Path: ${path} does not begin with a path variable" + echo "Path: ${path} does not begin with a path variable" exitcode = 1 } } if (exitcode > 0) { error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" + } else { + echo "ISM file: ${ISMfile} - ok" } } From 08a7350ee8eed79257c89a310a10de6580d04d8c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 08:39:15 +0100 Subject: [PATCH 449/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 823c117..51edd3d 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -31,9 +31,8 @@ def placeBinaries(artifactDetails, installerDetails) { } @NonCPS -def checkISMfile(String ISMfile) { - def xml = readFile(ISMfile) - echo "Checking ISM file: ${ISMfile}" +def checkISMfilebackend(String xml) { + def String message = "" def parsedxml = new XmlParser(true, true, true).parseText(xml) def pathvariableinfo = [:] @@ -54,26 +53,31 @@ def checkISMfile(String ISMfile) { } // Check that each path variable is defined beginning with a path variable or the WORKSPACE one pathvariableinfo.each{ pair -> - echo "pair key: ${pair.key} value: ${pair.value}" + message += "pair key: ${pair.key} value: ${pair.value}\n" def path = pair.value def pathvarmatcher = path =~ /<(.*)>.*/ if (pathvarmatcher.matches()) { def pathvarfound = pathvarmatcher.group(1) if (pathvarfound != 'WORKSPACE') { if (!pathvariableinfo.containsKey(pathvarfound)) { - echo "Undefined path variable ${pathvarfound} found" + message += "Undefined path variable ${pathvarfound} found\n" exitcode = 1 } } } else { - echo "Path: ${path} does not begin with a path variable" + message += "Path: ${path} does not begin with a path variable\n" exitcode = 1 } } + return [exitcode, message] +} + +def checkISMfile(String ISMfile) { + echo "Checking ISM file: ${ISMfile}" + def xml = readFile(ISMfile) + (exitcode, message) = checkISMfilebackend(xml) if (exitcode > 0) { - error "The ISM file: ${ISMfile} contains a path that is either absolute or does not begin with another path variable definition" - } else { - echo "ISM file: ${ISMfile} - ok" + error "ISM file errors found:\n${message}" } } From b7c5f9805b841417dc59a28c2a1fa96fb8374fcf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 08:57:32 +0100 Subject: [PATCH 450/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 51edd3d..6ac2e80 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -42,10 +42,10 @@ def checkISMfilebackend(String xml) { // Ignore the workspace definition which should have the only absolute path parsedxml.each{ table -> if (table.@name == "ISPathVariable") { - table.row.each{ row -> - def pathname = row.td[0].value()[0] - def path = row.td[1].value()[0] - if (path && pathname != "WORKSPACE") { + table['row'].each{ row -> + def pathname = row['td'][0].value()[0] + def path = row['td'][1].value()[0] + if (path && pathname != 'WORKSPACE') { pathvariableinfo[pathname] = path } } From f141d804103ce3301c18dbe79cdeaf10fd61d188 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 09:30:24 +0100 Subject: [PATCH 451/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index 6ac2e80..ca9c502 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -78,6 +78,9 @@ def checkISMfile(String ISMfile) { (exitcode, message) = checkISMfilebackend(xml) if (exitcode > 0) { error "ISM file errors found:\n${message}" + } else { + echo "Checked ISM file: ${ISMfile} - ok" + echo message } } From 2317111f3bde99ef2feae3a6feaaade1dd78868d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 9 Apr 2019 10:44:43 +0100 Subject: [PATCH 452/558] Perform installer file (ISM) checking --- install/installhelper.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/installhelper.groovy b/install/installhelper.groovy index ca9c502..1464e9d 100644 --- a/install/installhelper.groovy +++ b/install/installhelper.groovy @@ -76,11 +76,11 @@ def checkISMfile(String ISMfile) { echo "Checking ISM file: ${ISMfile}" def xml = readFile(ISMfile) (exitcode, message) = checkISMfilebackend(xml) + echo message if (exitcode > 0) { - error "ISM file errors found:\n${message}" + error "ISM file errors found" } else { - echo "Checked ISM file: ${ISMfile} - ok" - echo message + echo "Checked ISM file - ok" } } From 64f742ffa7aca162745f986688280725cc543f5d Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 12 Apr 2019 14:55:49 +0100 Subject: [PATCH 453/558] Remove diff folder before building --- copy/logic.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..0b4823d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -14,6 +14,7 @@ pipeline { cleanfolders = [ "${WORKSPACE}/logic", "${WORKSPACE}/embedded_logic", + "${WORKSPACE}/diff", ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) From 019ec1ab3b6e3c571cd2c82fe42adddbfe2f10a9 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 15 Apr 2019 09:57:18 +0100 Subject: [PATCH 454/558] Logic copy doesn't work if pre-existing folders from previous builds exist --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 6e97510..5a0c80d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -36,7 +36,7 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf meta + rm -v -rf diff meta embedded_logic mv -v logic/* . """ archiveArtifacts "${artifactlist}" From 4bb8bcec23a4ced93535ab7d4f19538568b30ba1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 7 May 2019 10:28:39 +0100 Subject: [PATCH 455/558] Warn of git stripping not being supported in Windows --- versioncontrol/git.groovy | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index f72fc5a..b639c91 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -170,13 +170,17 @@ boolean checkAllVersionsAgree(java.util.LinkedHashMap versions) { def stripgithistory(String rootfolder) { echo "Stripping git history out of ${rootfolder}" dir(rootfolder) { - sh """#!/bin/bash - pwd - find . -type d -name ".git" | xargs -iXX rm -rf XX - find . -name ".gitignore" | xargs -iXX rm -rf XX - find . -name ".gitmodules" | xargs -iXX rm -rf XX - find . -iname ".gitattributes" | xargs -iXX rm -rf XX - """ + if (isUnix()) { + sh """#!/bin/bash + pwd + find . -type d -name ".git" | xargs -iXX rm -rf XX + find . -name ".gitignore" | xargs -iXX rm -rf XX + find . -name ".gitmodules" | xargs -iXX rm -rf XX + find . -iname ".gitattributes" | xargs -iXX rm -rf XX + """ + } else { + echo "Not supporting git stripping in Windows at the moment" + } } } From a6fe97d65e5b7c3c3a45ce53ff4f6f150b6b931c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 19:39:52 +0100 Subject: [PATCH 456/558] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 23 ++++++++++++++++++++++- publish/do.groovy | 13 +++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 34707bc..9620e8a 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -42,7 +42,7 @@ def rewritemetafileasreleased(meta, String filepath) { meta.saveJsonSerialised(filepath) } -def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { +def hierarchyartifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" @@ -76,6 +76,27 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, return importedmeta } +def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { + def rooturl = repo.trim() + def rootrevision = revision.trim() + echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) + + def importedmeta = meta.importrelated(destpath) + + gh.stripgithistory(destpath) + def metafilepath = "${destpath}/${meta.metafilepath()}" + echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + rewritemetafileasreleased(meta, metafilepath) + + if (incrementversionnumber) { + meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) + } + return importedmeta +} + def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '**', String excludes = '', boolean fingerprint = false) { switch (source) { case "Jenkins job artifact": diff --git a/publish/do.groovy b/publish/do.groovy index 1599ca8..6207380 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,6 +148,19 @@ def convertitemstopublish(String itemstopublishstring) { return paths } +def convertstringitemstopublish(String itemstopublishstring) { + def List paths = [] + + def String[] items = itemstopublishstring.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { From 907a0f00e6b488f167c7ecd5a0a46f708ca29b6c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:04:39 +0100 Subject: [PATCH 457/558] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 9620e8a..d85818f 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) { + def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) From a32fbee67b59dba3c928b67fe2aa80b1b838b101 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:25:53 +0100 Subject: [PATCH 458/558] MCS-425 - Perform a sparse checkout of items to import from a release repo --- import/importer.groovy | 2 +- publish/do.groovy | 13 ------------- versioncontrol/git.groovy | 13 +++++++++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index d85818f..764618c 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -81,7 +81,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def rootrevision = revision.trim() echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" - def ArrayList sparseitemlist = gh.convertstringitemstopublish(filter) + def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) def importedmeta = meta.importrelated(destpath) diff --git a/publish/do.groovy b/publish/do.groovy index 6207380..1599ca8 100644 --- a/publish/do.groovy +++ b/publish/do.groovy @@ -148,19 +148,6 @@ def convertitemstopublish(String itemstopublishstring) { return paths } -def convertstringitemstopublish(String itemstopublishstring) { - def List paths = [] - - def String[] items = itemstopublishstring.split(',') - def String allpaths = "" - for(String item : items) { - paths.add(item) - allpaths += "${path}\n" - } - echo "Selected item to publish:\n${allpaths}" - return paths -} - // Publish selected items from a release to the customer folder on the ftp site // itemstopublishstring is a comma-seperated set of strings in this format ## def publishhierarchy(releasemeta, String itemstopublishstring, String customer, String softwarerelease, String maxdepth) { diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 534084d..818734a 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -246,6 +246,19 @@ def gitsparserestore(String targetDir) { } } +def List convertSparseItemsString(String itemstopublish) { + def List paths = [] + + def String[] items = itemstopublish.split(',') + def String allpaths = "" + for(String item : items) { + paths.add(item) + allpaths += "${path}\n" + } + echo "Selected item to publish:\n${allpaths}" + return paths +} + def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { echo "Performing alternative checkout" def String reporootpath = lfsinstallandreporoot(gitURL) From 03046f5c04c23f17a14d6e46bac38d2bf71d4e70 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 23 May 2019 23:33:55 +0100 Subject: [PATCH 459/558] MCS-425 - Perform a sparse checkout of items to import from a release repo --- versioncontrol/git.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index 818734a..811d072 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -247,16 +247,16 @@ def gitsparserestore(String targetDir) { } def List convertSparseItemsString(String itemstopublish) { - def List paths = [] + def List sparsepaths = [] def String[] items = itemstopublish.split(',') def String allpaths = "" for(String item : items) { - paths.add(item) - allpaths += "${path}\n" + sparsepaths.add(item) + allpaths += "${item}\n" } echo "Selected item to publish:\n${allpaths}" - return paths + return sparsepaths } def gitCheckoutAlt(String gitURL, String revision, String targetDir, String repoRootSuffix = "", int cloneDepth = 1, boolean shallowClone = true, ArrayList sparseitemlist = new ArrayList()) { From af12802f1b0661ffe15ba79dacfd058a8e1e0e6a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 13:30:15 +0100 Subject: [PATCH 460/558] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- meta/folder.groovy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index 764618c..fd43379 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -68,7 +68,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) diff --git a/meta/folder.groovy b/meta/folder.groovy index b16db5d..f8c033b 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -491,6 +491,7 @@ class MetaInfo extends GISJsonSerialised { } void markreleased() { + script.echo("Marking product: ${productinfo.productname} component: ${productinfo.subcomponentname} as released") releaseinfo.markreleased() for (associated in associatedmeta) { associated.meta.releaseinfo.markreleased() From 78f45274fa261ec1c0b7200a0e62835a9b2417a1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 6 Jun 2019 14:08:13 +0100 Subject: [PATCH 461/558] One level of meta too high was being marked as released when importing an already released component (from a release repository) --- import/importer.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/import/importer.groovy b/import/importer.groovy index fd43379..09eb337 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -89,7 +89,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" echo "Marking root meta as released and rewriting meta file: ${metafilepath}" - rewritemetafileasreleased(meta, metafilepath) + rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { meta.buildstampinfo.changerevision(versiongenerator.incrementrevision(meta.buildstampinfo.revision, importedmeta.buildstampinfo.revision)) From 84ff4211df10e319bda246902649d5f99fe85b11 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 18 Jul 2019 11:22:06 +0100 Subject: [PATCH 462/558] Provide logging to debug why import of Print Server artifacts results in duff associated meta data --- meta/folder.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index f8c033b..b559458 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -571,6 +571,7 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJson(String filepath) { + script.echo("Deserialising JSON at file: ${filepath}" def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } @@ -617,6 +618,8 @@ class MetaInfo extends GISJsonSerialised { def importedmetafile = adjustedpath + "/" + metafilepath() if (script.fileExists(importedmetafile)) { mi.deserialiseJson(importedmetafile) + } else { + script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") } RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) From 4ea4d19eaa1227e15ea2d92622fb7da288bb0ed7 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 18 Jul 2019 11:34:23 +0100 Subject: [PATCH 463/558] Provide logging to debug why import of Print Server artifacts results in duff associated meta data --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index b559458..ddc99c4 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -571,7 +571,7 @@ class MetaInfo extends GISJsonSerialised { } def deserialiseJson(String filepath) { - script.echo("Deserialising JSON at file: ${filepath}" + script.echo("Deserialising JSON at file: ${filepath}") def rawmeta = script.readJSON(file: filepath) deserialiseJson(rawmeta) } From dac329445c05a512a1502bc69808ca132ff28488 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:43:21 +0100 Subject: [PATCH 464/558] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 738c04a..1898594 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -92,9 +92,10 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { } } -def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { +def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") + bool exceptionduringcopyartifacts = false for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -102,22 +103,30 @@ def getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, Strin } // Copy relevant artifacts to temp folder echo "Copying artifacts: ${filterpath} to ${tempfolder}" - copyArtifacts( - filter: filterpath, - projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, - selector: [ - $class: 'ParameterizedBuildSelector', - parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" - ], - target: tempfolder - ) - // Move relevant artifacts from temp folder to release position - def originalrootdir = tempfolder + "/" + buildinfo.offsetpath - dir(originalrootdir) { - sh("mv * ${repofolder}") + try { + copyArtifacts( + filter: filterpath, + projectName: toplevelbuildmeta.jenkinsbuildinfo.buildjobname, + selector: [ + $class: 'ParameterizedBuildSelector', + parameterName: "${toplevelbuildmeta.jenkinsbuildinfo.buildnumber}" + ], + target: tempfolder + ) + } catch (Exception e) { + echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" + exceptionduringcopyartifacts = true + summaryappend("Does not exist: ${artifactfilter}," + } else { + // Move relevant artifacts from temp folder to release position + def originalrootdir = tempfolder + "/" + buildinfo.offsetpath + dir(originalrootdir) { + sh("mv * ${repofolder}") + } } summaryappend("${artifactfilter},") } + return exceptionduringcopyartifacts } def releasegitadd() { @@ -357,9 +366,9 @@ def releasecomponent(toplevelbuildmeta, buildinfo, String otherreleasetags, Stri checkoutsourcerepo(buildmeta, uniquefolderforsourcerepo) - getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp) - - gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + if (getcomponentartifacts(toplevelbuildmeta, buildinfo, uniquefolderforrepo, uniquefoldertemp)) { + gitrelease(buildmeta, previousreleasemeta, uniquefolderforrepo, uniquefolderforsourcerepo, otherreleasetags, dryrunpush) + } summaryappend("\n") } From 32952f0677b254f66df6ec98a6ef2c5b28796ffe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:51:29 +0100 Subject: [PATCH 465/558] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 1898594..10fc165 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -117,7 +117,8 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true summaryappend("Does not exist: ${artifactfilter}," - } else { + } + if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { From 152eb536a4ef826e5c183713ca29fcb5a940ccce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 13:53:31 +0100 Subject: [PATCH 466/558] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/do.groovy b/release/do.groovy index 10fc165..c1c4277 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -116,7 +116,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true - summaryappend("Does not exist: ${artifactfilter}," + summaryappend("Does not exist: ${artifactfilter},") } if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position From d52f4538de6630627a60b4106f9b4b2a92ede79e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 14:09:53 +0100 Subject: [PATCH 467/558] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index c1c4277..0e7809e 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -95,7 +95,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - bool exceptionduringcopyartifacts = false + boolean exceptionduringcopyartifacts = false for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -116,7 +116,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" exceptionduringcopyartifacts = true - summaryappend("Does not exist: ${artifactfilter},") + summaryappend("Does not exist:") } if (!exceptionduringcopyartifacts) { // Move relevant artifacts from temp folder to release position From bf72927c93ee67dba6c2871de449ab99f2503931 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 19 Jul 2019 14:33:01 +0100 Subject: [PATCH 468/558] Don't release a component if it is not available in the build job's artifacts --- release/do.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/release/do.groovy b/release/do.groovy index 0e7809e..1b21bcf 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -95,7 +95,7 @@ def checkoutsourcerepo(buildmeta, String checkoutfolder) { def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolder, String tempfolder) { // Copy relevant artifacts to temp folder summaryappend("Artifacts: ") - boolean exceptionduringcopyartifacts = false + def boolean result = true for (artifactfilter in buildinfo.meta.releaseinfo.releaseitems) { def filterpath = artifactfilter if (buildinfo.offsetpath) { @@ -115,10 +115,10 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde ) } catch (Exception e) { echo "Exception occurred while copying artifacts from ${toplevelbuildmeta.jenkinsbuildinfo.buildjobname}: ${e.getMessage()}" - exceptionduringcopyartifacts = true + result = false summaryappend("Does not exist:") } - if (!exceptionduringcopyartifacts) { + if (result) { // Move relevant artifacts from temp folder to release position def originalrootdir = tempfolder + "/" + buildinfo.offsetpath dir(originalrootdir) { @@ -127,7 +127,7 @@ def boolean getcomponentartifacts(toplevelbuildmeta, buildinfo, String repofolde } summaryappend("${artifactfilter},") } - return exceptionduringcopyartifacts + return result } def releasegitadd() { From f0ef516b8e5383d047524bff65c0e6c6067eaeda Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 15:12:50 +0100 Subject: [PATCH 469/558] HMBXR5601-827 - Update compile output suitable for packaging --- compile/eclipsebuild/eclipseds5build.groovy | 27 +++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 302b4f8..e93ed4b 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -54,7 +54,7 @@ def doCheck(checkpathlist, build_variant = "") { return result } -def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { +def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") { def result = 0 def destFolder = "" def srcFile = "" @@ -71,9 +71,12 @@ def doExport(exportfolder, exportpathlist, project = "", build_variant = "") { } else { echo "No build variant" - destFolder = "${exportfolder}/${project}" + destFolder = "${exportfolder}" srcFileExpanded = "${WORKSPACE}/${exportpath}" } + if (partdest) { + destFolder += "/" + partdest + } } dir (destFolder) { echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" @@ -146,7 +149,7 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } // Build and export files -def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}/bin") { +def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}/export") { toBuild = [] if (variantstobuild) { toBuildUntrimmed = variantstobuild.split(',') @@ -162,32 +165,26 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) } for (appInfo in apps ) { - (build, cleanbuild, project, variants, exportpathlist) = appInfo + (build, project, variants, exportpathlist, destpart) = appInfo if (build) { - if (cleanbuild) { - buildtype = "-cleanBuild" - } - else { - buildtype = "-build" - } for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) try { if (toBuild.contains(build_variant)) { doDisplayLeftovers() doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\'" + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\'" timeout(time: 10, unit: 'MINUTES') { if (isUnix()) { sh """ echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer ${buildtype} \"${project}/${build_variant}\"\' + ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\' echo End time: `date` """ } else { bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer ${buildtype} "${project}\\${build_variant}" + "%ds5shellcommand%" "%buildcommand%" -no-indexer -build "${project}\\${build_variant}" """ } } @@ -195,7 +192,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou if (result != 0) { error "Failed to check files...." } - result = doExport(exportfolder, exportpathlist, "", build_variant) + result = doExport(exportfolder, exportpathlist, destpart, build_variant) if (result != 0) { error "Failed to copy...." } @@ -214,7 +211,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou else { // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { - result = doExport(exportfolder, exportpathlist, project) + result = doExport(exportfolder, exportpathlist, destpart) if (result != 0) { error "Failed to copy...." } From 17ddcb8404e77127c2dae4cb22e327ecf622b955 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 15:38:21 +0100 Subject: [PATCH 470/558] HMBXR5601-827 - Update compile output suitable for packaging --- compile/eclipsebuild/eclipseds5build.groovy | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index e93ed4b..bb79b36 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -79,10 +79,11 @@ def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") { } } dir (destFolder) { - echo "Copy \"${srcFileExpanded}\" to \"${destFolder}\"" + def destFolderExpanded = destFolder.replace("-BUILDVARIANT-", "${build_variant}") + echo "Copy \"${srcFileExpanded}\" to \"${destFolderExpanded}\"" if (isUnix()) { result = sh( script: """ - cp -r -v "${srcFileExpanded}" "${destFolder}" + cp -r -v "${srcFileExpanded}" "${destFolderExpanded}" if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true @@ -93,7 +94,7 @@ def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") { } else { result = bat( script: """ - copy "${srcFileExpanded}" "${destFolder}" + copy "${srcFileExpanded}" "${destFolderExpanded}" if errorlevel 1 EXIT /b 1 """, returnStatus: true @@ -209,12 +210,20 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } else { - // Only export if the project has no variants - signifying that there are files to export which have been prebuilt if (!variants) { result = doExport(exportfolder, exportpathlist, destpart) if (result != 0) { error "Failed to copy...." } + } else { + for (build_variant in variants) { + if (toBuild.contains(build_variant)) { + result = doExport(exportfolder, exportpathlist, destpart, build_variant) + if (result != 0) { + error "Failed to copy...." + } + } + } } } } From d123e7f152509ce599b8b353b9a431186ffd7d51 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 15:49:24 +0100 Subject: [PATCH 471/558] HMBXR5601-827 - Update compile output suitable for packaging --- compile/eclipsebuild/eclipseds5build.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index bb79b36..9ebe541 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -78,8 +78,8 @@ def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") { destFolder += "/" + partdest } } - dir (destFolder) { - def destFolderExpanded = destFolder.replace("-BUILDVARIANT-", "${build_variant}") + def destFolderExpanded = destFolder.replace("-BUILDVARIANT-", "${build_variant}") + dir (destFolderExpanded) { echo "Copy \"${srcFileExpanded}\" to \"${destFolderExpanded}\"" if (isUnix()) { result = sh( script: """ From 36c28665e08be410e640d07bbce2d3ddb1ea7839 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 16:05:02 +0100 Subject: [PATCH 472/558] HMBXR5601-827 - Update compile output suitable for packaging --- compile/eclipsebuild/eclipseds5build.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 9ebe541..b3e25bd 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -62,18 +62,16 @@ def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") { for (exportpath in exportpathlist) { echo "Export path at: ${exportpath}" script { - echo "Build variant is ${build_variant}" if (build_variant) { echo "Build variant is ${build_variant}" - destFolder = "${exportfolder}/${build_variant}" srcFile = "${WORKSPACE}/${exportpath}" srcFileExpanded = srcFile.replace("-BUILDVARIANT-", "${build_variant}") } else { echo "No build variant" - destFolder = "${exportfolder}" srcFileExpanded = "${WORKSPACE}/${exportpath}" } + destFolder = "${exportfolder}" if (partdest) { destFolder += "/" + partdest } @@ -165,7 +163,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou addToolchainPath(toolchainfolder) doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) } - for (appInfo in apps ) { + for (appInfo in apps) { (build, project, variants, exportpathlist, destpart) = appInfo if (build) { for (build_variant in variants) { From 3c7deff68c6fe5bd2adb87cfdab8d4c6a924985b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 16:20:48 +0100 Subject: [PATCH 473/558] HMBXR5601-827 - Update compile output suitable for packaging --- compile/eclipsebuild/eclipseds5build.groovy | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index b3e25bd..721a024 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -164,7 +164,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) } for (appInfo in apps) { - (build, project, variants, exportpathlist, destpart) = appInfo + (build, export, project, variants, exportpathlist, destpart) = appInfo if (build) { for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) @@ -191,9 +191,11 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou if (result != 0) { error "Failed to check files...." } - result = doExport(exportfolder, exportpathlist, destpart, build_variant) - if (result != 0) { - error "Failed to copy...." + if (export) { + result = doExport(exportfolder, exportpathlist, destpart, build_variant) + if (result != 0) { + error "Failed to copy...." + } } } else { @@ -208,17 +210,19 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } else { - if (!variants) { - result = doExport(exportfolder, exportpathlist, destpart) - if (result != 0) { - error "Failed to copy...." - } - } else { - for (build_variant in variants) { - if (toBuild.contains(build_variant)) { - result = doExport(exportfolder, exportpathlist, destpart, build_variant) - if (result != 0) { - error "Failed to copy...." + if (export) { + if (!variants) { + result = doExport(exportfolder, exportpathlist, destpart) + if (result != 0) { + error "Failed to copy...." + } + } else { + for (build_variant in variants) { + if (toBuild.contains(build_variant)) { + result = doExport(exportfolder, exportpathlist, destpart, build_variant) + if (result != 0) { + error "Failed to copy...." + } } } } From 0ae452e1ba613b9ee09aff2d88653902ad701b52 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 16:43:26 +0100 Subject: [PATCH 474/558] HMBXR5601-827 - Update compile output suitable for packaging --- compile/eclipsebuild/eclipseds5build.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 721a024..77124d7 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -164,7 +164,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) } for (appInfo in apps) { - (build, export, project, variants, exportpathlist, destpart) = appInfo + (build, doexport, project, variants, exportpathlist, destpart) = appInfo if (build) { for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) @@ -191,7 +191,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou if (result != 0) { error "Failed to check files...." } - if (export) { + if (doexport) { result = doExport(exportfolder, exportpathlist, destpart, build_variant) if (result != 0) { error "Failed to copy...." @@ -210,7 +210,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } else { - if (export) { + if (doexport) { if (!variants) { result = doExport(exportfolder, exportpathlist, destpart) if (result != 0) { From 54e5d7ff081b06795b2578ffb2a678aae63529c5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 16:52:24 +0100 Subject: [PATCH 475/558] HMBXR5601-827 - Update compile output suitable for packaging - adding debug --- compile/eclipsebuild/eclipseds5build.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 77124d7..e3d2ffb 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -191,6 +191,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou if (result != 0) { error "Failed to check files...." } + echo "Doing export: ${doexport}" if (doexport) { result = doExport(exportfolder, exportpathlist, destpart, build_variant) if (result != 0) { @@ -202,7 +203,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou echo "Not building ${build_variant} variant of project: ${project}" } } catch (Exception e) { - echo "Build failed - Waiting for a while" + echo "Exception: ${e.message()}\nBuild failed - Waiting for a while" sleep(time: 4, unit: 'SECONDS') error "Build failed..." } @@ -210,6 +211,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } else { + echo "Doing export: ${doexport}" if (doexport) { if (!variants) { result = doExport(exportfolder, exportpathlist, destpart) @@ -231,7 +233,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } } catch (Exception e) { // doKillLeftovers() - error "Build failed - Killing leftovers" + error "Exception: ${e.message()}\nBuild failed - Killing leftovers" } } } From fbd1b756987f7533d70714c9174ea0aa1aad6da8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 17:32:08 +0100 Subject: [PATCH 476/558] HMBXR5601-827 - Update compile output suitable for packaging - adding debug --- compile/eclipsebuild/eclipseds5build.groovy | 133 +++++++++++--------- 1 file changed, 71 insertions(+), 62 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index e3d2ffb..b5b5401 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -147,6 +147,74 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } } +def doBuildAndExportProject(String project, boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { + echo "Doing build and export project: ${project} : ${doexport}" + for (build_variant in variants) { + retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) + try { + if (toBuild.contains(build_variant)) { + doDisplayLeftovers() + doRegisterToolchain(ds5folder, toolchainfolder) + echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\'" + timeout(time: 10, unit: 'MINUTES') { + if (isUnix()) { + sh """ + echo Start time: `date` + ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\' + echo End time: `date` + """ + } + else { + bat """ + "%ds5shellcommand%" "%buildcommand%" -no-indexer -build "${project}\\${build_variant}" + """ + } + } + result = doCheck(exportpathlist, build_variant) + if (result != 0) { + error "Failed to check files...." + } + echo "Doing export: ${doexport}" + if (doexport) { + result = doExport(exportfolder, exportpathlist, destpart, build_variant) + if (result != 0) { + error "Failed to copy...." + } + } + } + else { + echo "Not building ${build_variant} variant of project: ${project}" + } + } catch (Exception e) { + echo "Exception: ${e.getMessage()}\nBuild failed - Waiting for a while" + sleep(time: 4, unit: 'SECONDS') + error "Build failed..." + } + } + } +} + +def doExportOnlyProject(String project, boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { + echo "Doing export only project: ${project} : ${doexport}" + if (doexport) { + if (!variants) { + result = doExport(exportfolder, exportpathlist, destpart) + if (result != 0) { + error "Failed to copy...." + } + } else { + for (build_variant in variants) { + if (toBuild.contains(build_variant)) { + result = doExport(exportfolder, exportpathlist, destpart, build_variant) + if (result != 0) { + error "Failed to copy...." + } + } + } + } + } +} + // Build and export files def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder, variantstobuild, exportfolder = "${WORKSPACE}/export") { toBuild = [] @@ -166,74 +234,15 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (appInfo in apps) { (build, doexport, project, variants, exportpathlist, destpart) = appInfo if (build) { - for (build_variant in variants) { - retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - try { - if (toBuild.contains(build_variant)) { - doDisplayLeftovers() - doRegisterToolchain(ds5folder, toolchainfolder) - echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { - if (isUnix()) { - sh """ - echo Start time: `date` - ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\' - echo End time: `date` - """ - } - else { - bat """ - "%ds5shellcommand%" "%buildcommand%" -no-indexer -build "${project}\\${build_variant}" - """ - } - } - result = doCheck(exportpathlist, build_variant) - if (result != 0) { - error "Failed to check files...." - } - echo "Doing export: ${doexport}" - if (doexport) { - result = doExport(exportfolder, exportpathlist, destpart, build_variant) - if (result != 0) { - error "Failed to copy...." - } - } - } - else { - echo "Not building ${build_variant} variant of project: ${project}" - } - } catch (Exception e) { - echo "Exception: ${e.message()}\nBuild failed - Waiting for a while" - sleep(time: 4, unit: 'SECONDS') - error "Build failed..." - } - } - } + doBuildAndExportProject(project, doexport, variants, exportfolder, exportpathlist, destpart) } else { - echo "Doing export: ${doexport}" - if (doexport) { - if (!variants) { - result = doExport(exportfolder, exportpathlist, destpart) - if (result != 0) { - error "Failed to copy...." - } - } else { - for (build_variant in variants) { - if (toBuild.contains(build_variant)) { - result = doExport(exportfolder, exportpathlist, destpart, build_variant) - if (result != 0) { - error "Failed to copy...." - } - } - } - } - } + doExportOnlyProject(project, doexport, variants, exportfolder, exportpathlist, destpart) } } } catch (Exception e) { // doKillLeftovers() - error "Exception: ${e.message()}\nBuild failed - Killing leftovers" + error "Exception: ${e.getMessage()}\nBuild failed - Killing leftovers" } } } From 5d081f74cfbe86f38c6e401723201c76277e9bcf Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 17:39:32 +0100 Subject: [PATCH 477/558] HMBXR5601-827 - Update compile output suitable for packaging - adding debug --- compile/eclipsebuild/eclipseds5build.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index b5b5401..0de368c 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -147,7 +147,7 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } } -def doBuildAndExportProject(String project, boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { +def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5shellcommand, String buildcommand, String project, boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { echo "Doing build and export project: ${project} : ${doexport}" for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) @@ -234,7 +234,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou for (appInfo in apps) { (build, doexport, project, variants, exportpathlist, destpart) = appInfo if (build) { - doBuildAndExportProject(project, doexport, variants, exportfolder, exportpathlist, destpart) + doBuildAndExportProject(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, project, doexport, variants, exportfolder, exportpathlist, destpart) } else { doExportOnlyProject(project, doexport, variants, exportfolder, exportpathlist, destpart) From 382a48c08abea0c5d108aebbfbbf335478071130 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 17:46:24 +0100 Subject: [PATCH 478/558] HMBXR5601-827 - Update compile output suitable for packaging - adding debug --- compile/eclipsebuild/eclipseds5build.groovy | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 0de368c..3070e4e 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -151,7 +151,7 @@ def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5 echo "Doing build and export project: ${project} : ${doexport}" for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) - try { +// try { if (toBuild.contains(build_variant)) { doDisplayLeftovers() doRegisterToolchain(ds5folder, toolchainfolder) @@ -185,11 +185,11 @@ def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5 else { echo "Not building ${build_variant} variant of project: ${project}" } - } catch (Exception e) { - echo "Exception: ${e.getMessage()}\nBuild failed - Waiting for a while" - sleep(time: 4, unit: 'SECONDS') - error "Build failed..." - } +// } catch (Exception e) { +// echo "Exception: ${e.getMessage()}\nBuild failed - Waiting for a while" +// sleep(time: 4, unit: 'SECONDS') +// error "Build failed..." +// } } } } @@ -226,7 +226,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { - try { +// try { timeout(time: 2, unit: 'MINUTES') { addToolchainPath(toolchainfolder) doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) @@ -240,10 +240,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doExportOnlyProject(project, doexport, variants, exportfolder, exportpathlist, destpart) } } - } catch (Exception e) { - // doKillLeftovers() - error "Exception: ${e.getMessage()}\nBuild failed - Killing leftovers" - } +// } catch (Exception e) { +// // doKillLeftovers() +// error "Exception: ${e.getMessage()}\nBuild failed - Killing leftovers" +// } } } From 709ab6b27699593dc8507e3bbca6fc17fb27d849 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 17:59:14 +0100 Subject: [PATCH 479/558] HMBXR5601-827 - Update compile output suitable for packaging - adding debug --- compile/eclipsebuild/eclipseds5build.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 3070e4e..779c235 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -147,7 +147,7 @@ def doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, } } -def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5shellcommand, String buildcommand, String project, boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { +def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5shellcommand, String buildcommand, String project, Boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { echo "Doing build and export project: ${project} : ${doexport}" for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) @@ -194,7 +194,7 @@ def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5 } } -def doExportOnlyProject(String project, boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { +def doExportOnlyProject(String project, Boolean doexport, variants, String exportfolder, exportpathlist, String destpart) { echo "Doing export only project: ${project} : ${doexport}" if (doexport) { if (!variants) { @@ -232,6 +232,8 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) } for (appInfo in apps) { + def Boolean build + def Boolean doexport (build, doexport, project, variants, exportpathlist, destpart) = appInfo if (build) { doBuildAndExportProject(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, project, doexport, variants, exportfolder, exportpathlist, destpart) From 5d2d494d20952d487b5795e848812ba9912a8da5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 19 Aug 2019 18:37:16 +0100 Subject: [PATCH 480/558] HMBXR5601-827 - Update compile output suitable for packaging - adding debug --- compile/eclipsebuild/eclipseds5build.groovy | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 779c235..9ab8df3 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -151,7 +151,7 @@ def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5 echo "Doing build and export project: ${project} : ${doexport}" for (build_variant in variants) { retry(maxprojectbuildretries) { // Eclipse occasionally raises exceptions (rather than the builds failing) -// try { + try { if (toBuild.contains(build_variant)) { doDisplayLeftovers() doRegisterToolchain(ds5folder, toolchainfolder) @@ -185,11 +185,11 @@ def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5 else { echo "Not building ${build_variant} variant of project: ${project}" } -// } catch (Exception e) { -// echo "Exception: ${e.getMessage()}\nBuild failed - Waiting for a while" -// sleep(time: 4, unit: 'SECONDS') -// error "Build failed..." -// } + } catch (Exception e) { + echo "Exception: ${e.getMessage()}\nBuild failed - Waiting for a while" + sleep(time: 4, unit: 'SECONDS') + error "Build failed..." + } } } } @@ -226,7 +226,7 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou } // The lock is being used because it was being found that the doKillLeftovers was affecting other builds lock(resource: "eclipsebuild_${env.NODE_NAME}") { -// try { + try { timeout(time: 2, unit: 'MINUTES') { addToolchainPath(toolchainfolder) doImportProjects(ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sourcefolder) @@ -242,10 +242,10 @@ def doBuild(apps, ds5folder, toolchainfolder, ds5shellcommand, buildcommand, sou doExportOnlyProject(project, doexport, variants, exportfolder, exportpathlist, destpart) } } -// } catch (Exception e) { -// // doKillLeftovers() -// error "Exception: ${e.getMessage()}\nBuild failed - Killing leftovers" -// } + } catch (Exception e) { + // doKillLeftovers() + error "Exception: ${e.getMessage()}\nBuild failed - Killing leftovers" + } } } From 9a7e394fa2517653ff1f7b4431c55d6e49634465 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Aug 2019 13:24:14 +0100 Subject: [PATCH 481/558] HMBXR5601-827 - Create debian package definition class --- packaging/debian.groovy | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 packaging/debian.groovy diff --git a/packaging/debian.groovy b/packaging/debian.groovy new file mode 100644 index 0000000..d55b6a9 --- /dev/null +++ b/packaging/debian.groovy @@ -0,0 +1,49 @@ +class DebRootDefinitions { + String PACKAGE_PREFIX + String PACKAGE_FOLDER_NAME + String PACKAGE_NAME + + String DEBPATH + + String ETC + String GIS_OPT_BIN + String GIS_OPT_LIB + String GIS_OPT_SCRIPTS_APP + String GIS_OPT_SCRIPTS_EL + String GIS_OPT_WWW + String GIS_OPT_OTHER + String GIS_OPT_LOGIC + String GIS_OPT_PRINTHEAD + + String USR_BIN + String USR_LIB_ARCH + String LIB_FIRMWARE + String HTTP_DEV_DIV + String LIB_SYSTEMD + + DebRootDefinitions(String rootpath, String packageprefix, String buildstamp) { + PACKAGE_PREFIX = packageprefix + PACKAGE_FOLDER_NAME = packageprefix + buildstamp + PACKAGE_NAME = packageprefix + buildstamp + "_armhf.deb" + + DEBPATH = "${rootpath}/${PACKAGE_FOLDER_NAME}" + + ETC = "${DEBPATH}/etc" + GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" + GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" + GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" + GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" + GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" + GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" + GIS_OPT_PRINTHEAD = "${DEBPATH}/opt/gis/printhead" + + USR_BIN = "${DEBPATH}/usr/bin" + USR_LIB_ARCH = "${DEBPATH}/usr/lib/arm-linux-gnueabihf" + LIB_FIRMWARE = "${DEBPATH}/lib/firmware" + HTTP_DEV_DIV = "${DEBPATH}/var/www/gis_dev" + LIB_SYSTEMD = "${DEBPATH}/lib/systemd" + } +} + +return this; \ No newline at end of file From 703c4930db6588ccca56d94bb8f5b74efe7e86e6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Aug 2019 13:39:05 +0100 Subject: [PATCH 482/558] HMBXR5601-827 - Create debian package definition class --- packaging/debian.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index d55b6a9..127173e 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -1,4 +1,4 @@ -class DebRootDefinitions { +class DebRootDefinition { String PACKAGE_PREFIX String PACKAGE_FOLDER_NAME String PACKAGE_NAME @@ -21,7 +21,7 @@ class DebRootDefinitions { String HTTP_DEV_DIV String LIB_SYSTEMD - DebRootDefinitions(String rootpath, String packageprefix, String buildstamp) { + DebRootDefinition(String rootpath, String packageprefix, String buildstamp) { PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix + buildstamp PACKAGE_NAME = packageprefix + buildstamp + "_armhf.deb" @@ -46,4 +46,8 @@ class DebRootDefinitions { } } +def DebRootDefinition debDefinition(String rootpath, String packageprefix, String buildstamp) { + return new DebRootDefinition(rootpath, packageprefix, buildstamp) +} + return this; \ No newline at end of file From 5891f4ec02f54ab96bd42e809982cda6562a5290 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Aug 2019 13:47:37 +0100 Subject: [PATCH 483/558] HMBXR5601-827 - Create debian package definition class --- packaging/debian.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 127173e..84ba090 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -3,6 +3,7 @@ class DebRootDefinition { String PACKAGE_FOLDER_NAME String PACKAGE_NAME + String PACKAGE_SOURCE_FOLDER_NAME String DEBPATH String ETC @@ -26,6 +27,7 @@ class DebRootDefinition { PACKAGE_FOLDER_NAME = packageprefix + buildstamp PACKAGE_NAME = packageprefix + buildstamp + "_armhf.deb" + PACKAGE_SOURCE_FOLDER_NAME = "${rootpath}/${PACKAGE_PREFIX}" DEBPATH = "${rootpath}/${PACKAGE_FOLDER_NAME}" ETC = "${DEBPATH}/etc" From 93430a782e82af1e64ffd0bafc067cf4f0850efe Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Aug 2019 13:59:08 +0100 Subject: [PATCH 484/558] HMBXR5601-827 - Create debian package definition class --- packaging/debian.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 84ba090..9340180 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -1,4 +1,5 @@ class DebRootDefinition { + String ROOTPATH String PACKAGE_PREFIX String PACKAGE_FOLDER_NAME String PACKAGE_NAME @@ -23,12 +24,13 @@ class DebRootDefinition { String LIB_SYSTEMD DebRootDefinition(String rootpath, String packageprefix, String buildstamp) { + ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix + buildstamp PACKAGE_NAME = packageprefix + buildstamp + "_armhf.deb" - PACKAGE_SOURCE_FOLDER_NAME = "${rootpath}/${PACKAGE_PREFIX}" - DEBPATH = "${rootpath}/${PACKAGE_FOLDER_NAME}" + PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" + DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" ETC = "${DEBPATH}/etc" GIS_OPT_BIN = "${DEBPATH}/opt/gis/bin" From 6df15e3bf2f60d7daf349426c423574c13dfff7f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Aug 2019 14:42:12 +0100 Subject: [PATCH 485/558] HMBXR5601-827 - Create debian package definition class --- packaging/debian.groovy | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 9340180..a7de1f0 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -54,4 +54,30 @@ def DebRootDefinition debDefinition(String rootpath, String packageprefix, Strin return new DebRootDefinition(rootpath, packageprefix, buildstamp) } +def buildPackage(String packageout, DebRootDefinition packagedefs) { + sh """#!/bin/bash + # Exit if any command fails + set -e + + mkdir -v -p "${packagedefs.USR_BIN}" + mkdir -v -p "${packagedefs.USR_LIB_ARCH}" + mkdir -v -p "${packagedefs.LIB_FIRMWARE}" + + mkdir -v -p "${packagedefs.HTTP_DEV_DIV}" + mkdir -v -p "${packagedefs.LIB_SYSTEMD}/system" + + cp -v -r "${SOURCE_PATH}/hmb-xr-5601-source/presto_xr/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/preinst" + chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/postinst" + chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/prerm" + chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/postrm" + + mkdir -v -p "${packageout}" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + """ +} + return this; \ No newline at end of file From 99efb640c1e6aea041684c53cff28b3dea1df4e5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 22 Aug 2019 18:08:54 +0100 Subject: [PATCH 486/558] HMBXR5601-827 - Create debian package definition class --- packaging/debian.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index a7de1f0..25d06a2 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -12,6 +12,7 @@ class DebRootDefinition { String GIS_OPT_LIB String GIS_OPT_SCRIPTS_APP String GIS_OPT_SCRIPTS_EL + String GIS_OPT_SCRIPTS_CONTROL String GIS_OPT_WWW String GIS_OPT_OTHER String GIS_OPT_LOGIC @@ -37,6 +38,7 @@ class DebRootDefinition { GIS_OPT_LIB = "${DEBPATH}/opt/gis/lib" GIS_OPT_SCRIPTS_APP = "${DEBPATH}/opt/gis/scripts/app" GIS_OPT_SCRIPTS_EL = "${DEBPATH}/opt/gis/scripts/el" + GIS_OPT_SCRIPTS_CONTROL = "${DEBPATH}/opt/gis/scripts/control" GIS_OPT_WWW = "${DEBPATH}/opt/gis/www" GIS_OPT_OTHER = "${DEBPATH}/opt/gis/other" GIS_OPT_LOGIC = "${DEBPATH}/opt/gis/logic" From a8f357e51230763357a22b11939805b63f580ab3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Aug 2019 09:17:23 +0100 Subject: [PATCH 487/558] Add static analysis --- staticanalysis/cppcheck.groovy | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index b3b997e..18df1cb 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -57,9 +57,17 @@ pipeline { script { def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" dir(CPPCHECKFOLDER) { - bat """ - "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml - """ + if (isUnix()) { + sh """#!/bin/bash + set -e + + "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml + """ + } else { + bat """ + "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml + """ + } } recordIssues(tools: [ cppCheck(pattern: "**/cppcheck.xml", reportEncoding: 'UTF-8'), From bb4f70629ba6bf7f3058d608a8fad358fefb2f4a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Aug 2019 09:19:50 +0100 Subject: [PATCH 488/558] Add static analysis --- staticanalysis/cppcheck.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index 18df1cb..54b683a 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -57,6 +57,7 @@ pipeline { script { def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" dir(CPPCHECKFOLDER) { + echo "Executing: "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml" if (isUnix()) { sh """#!/bin/bash set -e From 24384288ed2784219716d0e6cb072be679a6214a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Aug 2019 09:21:39 +0100 Subject: [PATCH 489/558] Add static analysis --- staticanalysis/cppcheck.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/staticanalysis/cppcheck.groovy b/staticanalysis/cppcheck.groovy index 54b683a..59323c1 100644 --- a/staticanalysis/cppcheck.groovy +++ b/staticanalysis/cppcheck.groovy @@ -57,7 +57,7 @@ pipeline { script { def CPPCHECKFOLDER = "${STATICANALYSIS}/cppcheck" dir(CPPCHECKFOLDER) { - echo "Executing: "${CPPCHECK}" --xml --xml-version=2 "${SRC}" 2> cppcheck.xml" + echo "Executing: ${CPPCHECK} --xml --xml-version=2 ${SRC} 2> cppcheck.xml" if (isUnix()) { sh """#!/bin/bash set -e From b0a28652ef2ae8ed61ed660e8dc407707708f69b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Aug 2019 09:33:31 +0100 Subject: [PATCH 490/558] GIS-1621 - Update static analysis --- staticanalysis/ccm.groovy | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index 290c897..aa697a6 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -63,9 +63,16 @@ pipeline { script { def CCMOLDER = "${STATICANALYSIS}/ccm" dir(CCMOLDER) { - bat """ - "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml - """ + if (isUnix()) { + sh """#!/bin/bash + # Uses mono + "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + """ + } else { + bat """ + "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml + """ + } } recordIssues(tools: [ ccm(pattern: "**/ccm.xml", reportEncoding: 'UTF-8'), From 80cbc7c97d7042937d7f2a10f63f711bc79fc951 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Aug 2019 09:52:21 +0100 Subject: [PATCH 491/558] Echo ccm command --- staticanalysis/ccm.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index aa697a6..66a75e4 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -63,6 +63,7 @@ pipeline { script { def CCMOLDER = "${STATICANALYSIS}/ccm" dir(CCMOLDER) { + echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" if (isUnix()) { sh """#!/bin/bash # Uses mono From 7185c9f4ce98450db3400ecdd20bd373daac3f53 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 23 Aug 2019 21:51:00 +0100 Subject: [PATCH 492/558] Strip non C++ files before analysing --- staticanalysis/ccm.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/staticanalysis/ccm.groovy b/staticanalysis/ccm.groovy index 66a75e4..93e4711 100644 --- a/staticanalysis/ccm.groovy +++ b/staticanalysis/ccm.groovy @@ -66,6 +66,7 @@ pipeline { echo "Executing: ${CCM} ${SRC} /xml /nummetrics=100 > ccm.xml" if (isUnix()) { sh """#!/bin/bash + find "${SRC}" -type f ! -name "*.cpp" -a ! -name "*.h" -a ! -name "*.hpp" -a ! -name "*.c" -exec rm -r {} \; # Uses mono "${CCM}" "${SRC}" /xml /nummetrics=100 > ccm.xml """ From e3a3ef617673b14b47603a6b15baaa1895c5999a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 14:57:02 +0100 Subject: [PATCH 493/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 109 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 publish/package.groovy diff --git a/publish/package.groovy b/publish/package.groovy new file mode 100644 index 0000000..3efe70b --- /dev/null +++ b/publish/package.groovy @@ -0,0 +1,109 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Checkout dependencies') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + PYTHONURL = 'git@hydra:tools/python.git' + if (isUnix()) { + PYTHONBRANCH = "origin/3.6.3_linux" + } + else { + PYTHONBRANCH = "origin/3.6.3_windows" + } + PYTHONDIR = "${WORKSPACE}/python" + PYTHONROOTDIR = 'python' + configurationhelper.gitCheckoutAndRegister(meta, 'python', PYTHONURL, PYTHONBRANCH, PYTHONDIR, PYTHONROOTDIR) + if (isUnix()) { + env.PYTHON = "${PYTHONDIR}/usr/local/bin/python3" + } + else { + env.PYTHON = "${PYTHONDIR}/ver/python.exe" + } + } + } + } + stage('Clean exports') { + steps { + script { + PACKAGEDEST = "${WORKSPACE}/package" + cleanfolders = [ + PACKAGEDEST, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying package artifacts') { + steps { + script { + packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) + } + } + } + stage('Aptly environment') { + steps { + script { + echo "Aptly config file is ${APTLY_CONFIG_FILE}" + sh """!/#bin/bash + aptly config show + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + stage('Create repo - if not already there') { + steps { + script { + echo "Repo name is ${REPO_NAME}" + sh """!/#bin/bash + aptly repo -config="${APTLY_CONFIG_FILE}" show "${REPO_NAME}" + if [ \$? -ne 0 ]; then + echo "aptly repo ${REPO_NAME} does not exist; Creating it" + aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + fi + """ + } + } + } + stage('Create snapshot - if not already there') { + steps { + script { + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME}" + sh """!/#bin/bash + # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + """ + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + From 522ee178b12705242020547db45c139a455dd2a5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:10:03 +0100 Subject: [PATCH 494/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/package.groovy b/publish/package.groovy index 3efe70b..da6791c 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -47,6 +47,7 @@ pipeline { stage('Copying package artifacts') { steps { script { + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) } } From 47e4690e2d79e22f1cd8b8aa923f0f0cd9f7b249 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:11:54 +0100 Subject: [PATCH 495/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index da6791c..bf48231 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -56,7 +56,7 @@ pipeline { steps { script { echo "Aptly config file is ${APTLY_CONFIG_FILE}" - sh """!/#bin/bash + sh """#!/bin/bash -v aptly config show if [ \$? -ne 0 ]; then exit 1; fi """ @@ -67,7 +67,7 @@ pipeline { steps { script { echo "Repo name is ${REPO_NAME}" - sh """!/#bin/bash + sh """#!/bin/bash -v aptly repo -config="${APTLY_CONFIG_FILE}" show "${REPO_NAME}" if [ \$? -ne 0 ]; then echo "aptly repo ${REPO_NAME} does not exist; Creating it" @@ -84,7 +84,7 @@ pipeline { echo "Snapshot name is ${SNAPSHOT_NAME}" FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME}" - sh """!/#bin/bash + sh """#!/bin/bash -v # This errors if the snapshot exists already; Ignore it aptly snapshot -config="${APTLY_CONFIG_FILE}" create "${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" """ From 22ab61265cae4656125c728ac38ffaefb1c748b1 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:17:50 +0100 Subject: [PATCH 496/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/package.groovy b/publish/package.groovy index bf48231..8626a61 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -87,6 +87,7 @@ pipeline { sh """#!/bin/bash -v # This errors if the snapshot exists already; Ignore it aptly snapshot -config="${APTLY_CONFIG_FILE}" create "${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" show "${FULL_SNAPSHOT_NAME}" """ } } From e86106fb26690c70aa8d25a16ed10bb0af2c1fdb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:41:19 +0100 Subject: [PATCH 497/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/publish/package.groovy b/publish/package.groovy index 8626a61..5fe46e8 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -49,6 +49,7 @@ pipeline { script { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) + PACKAGE_FOLDER = "${PACKAGEDEST}/package" } } } @@ -78,6 +79,17 @@ pipeline { } } } + stage('Add/update package in snapshot') { + steps { + script { + echo "Package folder is ${PACKAGE_FOLDER}" + sh """#!/bin/bash -v + aptly snapshot -config="${APTLY_CONFIG_FILE}" add -force-replace "${PACKAGE_FOLDER}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } stage('Create snapshot - if not already there') { steps { script { @@ -86,8 +98,11 @@ pipeline { echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME}" sh """#!/bin/bash -v # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" aptly snapshot -config="${APTLY_CONFIG_FILE}" create "${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi aptly snapshot -config="${APTLY_CONFIG_FILE}" show "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi """ } } From ff5f214124bbf2f1a38d1ca1340d06b4130703c0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:42:28 +0100 Subject: [PATCH 498/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/package.groovy b/publish/package.groovy index 5fe46e8..253f22e 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -84,7 +84,7 @@ pipeline { script { echo "Package folder is ${PACKAGE_FOLDER}" sh """#!/bin/bash -v - aptly snapshot -config="${APTLY_CONFIG_FILE}" add -force-replace "${PACKAGE_FOLDER}" + aptly repo -config="${APTLY_CONFIG_FILE}" add -force-replace "${PACKAGE_FOLDER}" if [ \$? -ne 0 ]; then exit 1; fi """ } From 8a9f983b7bf05b2c3acbe0a1149c118228a9cc92 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:44:26 +0100 Subject: [PATCH 499/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/package.groovy b/publish/package.groovy index 253f22e..99a2bdf 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -84,7 +84,7 @@ pipeline { script { echo "Package folder is ${PACKAGE_FOLDER}" sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" add -force-replace "${PACKAGE_FOLDER}" + aptly repo -config="${APTLY_CONFIG_FILE}" -force-replace add "${PACKAGE_FOLDER}" if [ \$? -ne 0 ]; then exit 1; fi """ } From 387c3d08485020fc4fed16478fb77718180ec38e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:46:25 +0100 Subject: [PATCH 500/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/package.groovy b/publish/package.groovy index 99a2bdf..b3256a8 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -84,7 +84,7 @@ pipeline { script { echo "Package folder is ${PACKAGE_FOLDER}" sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" -force-replace add "${PACKAGE_FOLDER}" + aptly repo -config="${APTLY_CONFIG_FILE}" -force-replace add "${REPO_NAME}" "${PACKAGE_FOLDER}" if [ \$? -ne 0 ]; then exit 1; fi """ } From d2a12d523e5b1aa3bddfe9f543088405f022a670 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 27 Aug 2019 15:51:48 +0100 Subject: [PATCH 501/558] HMBXR5601-827 - Add package publish operations --- publish/package.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/package.groovy b/publish/package.groovy index b3256a8..eac8553 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -86,6 +86,8 @@ pipeline { sh """#!/bin/bash -v aptly repo -config="${APTLY_CONFIG_FILE}" -force-replace add "${REPO_NAME}" "${PACKAGE_FOLDER}" if [ \$? -ne 0 ]; then exit 1; fi + aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi """ } } From d5de03abb9e9e27ea59cf882e51e8fe26454d5c8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Aug 2019 09:22:15 +0100 Subject: [PATCH 502/558] HMBXR5601-827 - Publish package --- publish/package.groovy | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/publish/package.groovy b/publish/package.groovy index eac8553..8e91143 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -92,7 +92,7 @@ pipeline { } } } - stage('Create snapshot - if not already there') { + stage('Create snapshot') { steps { script { echo "Snapshot name is ${SNAPSHOT_NAME}" @@ -109,6 +109,21 @@ pipeline { } } } + stage('Publish snapshot') { + steps { + script { + echo "Snapshot name is ${SNAPSHOT_NAME}" + FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME}" + sh """#!/bin/bash -v + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } } post { always { From 106c5fd0ffa25e7a0ba8aca7891c682cb69304e8 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Aug 2019 09:29:06 +0100 Subject: [PATCH 503/558] HMBXR5601-827 - Publish package --- publish/package.groovy | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index 8e91143..2a84034 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -79,7 +79,20 @@ pipeline { } } } - stage('Add/update package in snapshot') { + stage('Drop published repo - if any') { + steps { + script { + sh """#!/bin/bash -v + # This errors if the published repo exists already; Ignore it + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + """ + } + } + } + + stage('Add/update package in repo') { steps { script { echo "Package folder is ${PACKAGE_FOLDER}" @@ -112,12 +125,7 @@ pipeline { stage('Publish snapshot') { steps { script { - echo "Snapshot name is ${SNAPSHOT_NAME}" - FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" - echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME}" sh """#!/bin/bash -v - # This errors if the published repo exists already; Ignore it - aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" if [ \$? -ne 0 ]; then exit 1; fi """ From 7be6867a66da564abeb02da42c5f0587432a8a73 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 28 Aug 2019 11:36:05 +0100 Subject: [PATCH 504/558] HMBXR5601-827 - Show more status; Handle component areas of published repos --- publish/package.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index 2a84034..00f5923 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -69,12 +69,14 @@ pipeline { script { echo "Repo name is ${REPO_NAME}" sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" show "${REPO_NAME}" + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" if [ \$? -ne 0 ]; then echo "aptly repo ${REPO_NAME} does not exist; Creating it" aptly repo -config="${APTLY_CONFIG_FILE}" create "${REPO_NAME}" if [ \$? -ne 0 ]; then exit 1; fi fi + aptly repo -config="${APTLY_CONFIG_FILE}" -with-packages show "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi """ } } @@ -116,7 +118,7 @@ pipeline { aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" aptly snapshot -config="${APTLY_CONFIG_FILE}" create "${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" if [ \$? -ne 0 ]; then exit 1; fi - aptly snapshot -config="${APTLY_CONFIG_FILE}" show "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" if [ \$? -ne 0 ]; then exit 1; fi """ } @@ -126,7 +128,11 @@ pipeline { steps { script { sh """#!/bin/bash -v - aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" -force-overwrite snapshot "${FULL_SNAPSHOT_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" list + if [ \$? -ne 0 ]; then exit 1; fi + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" if [ \$? -ne 0 ]; then exit 1; fi """ } From 8b39d2eb49e8b90a65af196b137a3309b83749f6 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 3 Sep 2019 17:22:13 +0100 Subject: [PATCH 505/558] HMBXR5601-827 - Change names of packages to be neutral and without buildstamps to make future jobs easier --- packaging/debian.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 25d06a2..57d84fb 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -27,8 +27,8 @@ class DebRootDefinition { DebRootDefinition(String rootpath, String packageprefix, String buildstamp) { ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix - PACKAGE_FOLDER_NAME = packageprefix + buildstamp - PACKAGE_NAME = packageprefix + buildstamp + "_armhf.deb" + PACKAGE_FOLDER_NAME = packageprefix + PACKAGE_NAME = packageprefix + "_armhf.deb" PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" From 02de57be7a966093947b6ae61bcf687b8695508e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 5 Sep 2019 08:35:37 +0100 Subject: [PATCH 506/558] HMBXR5601-827 - Use release tag rather than build number to make each package unique --- packaging/debian.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 57d84fb..9799ba9 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -24,7 +24,7 @@ class DebRootDefinition { String HTTP_DEV_DIV String LIB_SYSTEMD - DebRootDefinition(String rootpath, String packageprefix, String buildstamp) { + DebRootDefinition(String rootpath, String packageprefix) { ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix @@ -52,8 +52,8 @@ class DebRootDefinition { } } -def DebRootDefinition debDefinition(String rootpath, String packageprefix, String buildstamp) { - return new DebRootDefinition(rootpath, packageprefix, buildstamp) +def DebRootDefinition debDefinition(String rootpath, String packageprefix) { + return new DebRootDefinition(rootpath, packageprefix) } def buildPackage(String packageout, DebRootDefinition packagedefs) { From 2c654736a094d50636963ff05dde125282d342ce Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 5 Sep 2019 08:40:02 +0100 Subject: [PATCH 507/558] HMBXR5601-827 - Use release tag rather than build number to make each package unique --- meta/folder.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index ddc99c4..6967def 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -722,6 +722,10 @@ class MetaInfo extends GISJsonSerialised { return buildstampinfo.buildstamp + "_" + jenkinsbuildinfo.buildnumber } + String releasetagfordebpackage() { + return buildstampinfo.buildstamp + "-" + jenkinsbuildinfo.buildnumber + } + Map putwithshortestoffsetpath(Map releasemetacollection, String uniquestring, RelatedMetaInfo relatedmetainfo) { if (releasemetacollection.containsKey(uniquestring)) { def existingentry = releasemetacollection[uniquestring] From 512ba83729a8c3649a118a3c9350f0d1580291ed Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 5 Sep 2019 08:56:49 +0100 Subject: [PATCH 508/558] HMBXR5601-827 - Versionstamper.py is leaving excess files in package --- versionstamping/versionstamper.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 4c380a6..7a9c397 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -19,21 +19,15 @@ def main(argv): print("Converting incidences of {0} with {1} in file: {2}\n".format(pattern, substitute, filetoconvert)) - filecopy = filetoconvert + ".orig" - try: - shutil.move(filetoconvert, filecopy) - except FileNotFoundError: - print("Could not find file: {0}\n".format(filetoconvert)) - return 1 + with open(filetoconvert,'r') as c: + fileread = c.read() count = 0 with open(filetoconvert, 'x') as f: - with open(filecopy,'r') as c: - fileread = c.read() - print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) - converted = fileread.replace(pattern, substitute) - count = converted.count(substitute) - print("Confirmed %s instances of %s post-conversion"%(count,substitute)) + print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) + converted = fileread.replace(pattern, substitute) + count = converted.count(substitute) + print("Confirmed %s instances of %s post-conversion"%(count,substitute)) f.write(converted) if count == 0: From ac0b047a69e2524394f3031aff310e611a4d8f02 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 5 Sep 2019 09:00:42 +0100 Subject: [PATCH 509/558] HMBXR5601-827 - Versionstamper.py is leaving excess files in package --- versionstamping/versionstamper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versionstamping/versionstamper.py b/versionstamping/versionstamper.py index 7a9c397..1a65e6d 100644 --- a/versionstamping/versionstamper.py +++ b/versionstamping/versionstamper.py @@ -23,7 +23,7 @@ def main(argv): fileread = c.read() count = 0 - with open(filetoconvert, 'x') as f: + with open(filetoconvert, 'w') as f: print("Detected %s instances of %s pre-conversion"%(fileread.count(pattern),pattern)) converted = fileread.replace(pattern, substitute) count = converted.count(substitute) From 4dc0283ef0f445c8f0167df8da0f58f06df98954 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 6 Sep 2019 18:56:50 +0100 Subject: [PATCH 510/558] HMBXR5601-827 - Remove force command options --- publish/package.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index 00f5923..32cc9eb 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -99,7 +99,7 @@ pipeline { script { echo "Package folder is ${PACKAGE_FOLDER}" sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" -force-replace add "${REPO_NAME}" "${PACKAGE_FOLDER}" + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${PACKAGE_FOLDER}" if [ \$? -ne 0 ]; then exit 1; fi aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" if [ \$? -ne 0 ]; then exit 1; fi @@ -128,7 +128,7 @@ pipeline { steps { script { sh """#!/bin/bash -v - aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" -force-overwrite snapshot "${FULL_SNAPSHOT_NAME}" + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" if [ \$? -ne 0 ]; then exit 1; fi aptly publish -config="${APTLY_CONFIG_FILE}" list if [ \$? -ne 0 ]; then exit 1; fi From 193ddb19808e790de9702e5a9ac88fac3ff0d0bb Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 10:19:21 +0100 Subject: [PATCH 511/558] HMBXR5601-827 - Add lintian output to debian package --- packaging/debian.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 9799ba9..6b3426f 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -79,6 +79,8 @@ def buildPackage(String packageout, DebRootDefinition packagedefs) { dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + + lintian "${packageout}/${packagedefs.PACKAGE_NAME}" || true """ } From 3cbb3081af6d0ad940382a5c67f9ad64de13b19f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 10:34:22 +0100 Subject: [PATCH 512/558] HMBXR5601-827 - Change file permissions to satisfy lintian --- packaging/debian.groovy | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 6b3426f..eb53f44 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -56,11 +56,13 @@ def DebRootDefinition debDefinition(String rootpath, String packageprefix) { return new DebRootDefinition(rootpath, packageprefix) } -def buildPackage(String packageout, DebRootDefinition packagedefs) { +def buildPackage(String packageout, DebRootDefinition packagedefs, String password) { sh """#!/bin/bash # Exit if any command fails set -e + echo "${password}" | sudo -S echo + mkdir -v -p "${packagedefs.USR_BIN}" mkdir -v -p "${packagedefs.USR_LIB_ARCH}" mkdir -v -p "${packagedefs.LIB_FIRMWARE}" @@ -69,10 +71,9 @@ def buildPackage(String packageout, DebRootDefinition packagedefs) { mkdir -v -p "${packagedefs.LIB_SYSTEMD}/system" cp -v -r "${SOURCE_PATH}/hmb-xr-5601-source/presto_xr/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/preinst" - chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/postinst" - chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/prerm" - chmod -v 775 "${packagedefs.DEBPATH}/DEBIAN/postrm" + + sudo chmod -v -R 755 "${packagedefs.DEBPATH}" + sudo chown -v -R root: "${packagedefs.DEBPATH}" mkdir -v -p "${packageout}" dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" From f93a80884de140e1fdc4f88c6d5bcc404477aa20 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 10:49:44 +0100 Subject: [PATCH 513/558] HMBXR5601-827 - Change file permissions to satisfy lintian --- packaging/debian.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index eb53f44..3dda8eb 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -81,7 +81,7 @@ def buildPackage(String packageout, DebRootDefinition packagedefs, String passwo dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - lintian "${packageout}/${packagedefs.PACKAGE_NAME}" || true + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true """ } From a619adfd2aa5ac8931daa45a1f87e593d61f3132 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 12:40:47 +0100 Subject: [PATCH 514/558] HMBXR5601-827 - Change permissions after package build so that it can be cleaned on the next build --- packaging/debian.groovy | 52 ++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 3dda8eb..cef692f 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -57,32 +57,46 @@ def DebRootDefinition debDefinition(String rootpath, String packageprefix) { } def buildPackage(String packageout, DebRootDefinition packagedefs, String password) { - sh """#!/bin/bash - # Exit if any command fails - set -e + try { + sh """#!/bin/bash + # Exit if any command fails + set -e - echo "${password}" | sudo -S echo + echo "${password}" | sudo -S echo - mkdir -v -p "${packagedefs.USR_BIN}" - mkdir -v -p "${packagedefs.USR_LIB_ARCH}" - mkdir -v -p "${packagedefs.LIB_FIRMWARE}" + mkdir -v -p "${packagedefs.USR_BIN}" + mkdir -v -p "${packagedefs.USR_LIB_ARCH}" + mkdir -v -p "${packagedefs.LIB_FIRMWARE}" - mkdir -v -p "${packagedefs.HTTP_DEV_DIV}" - mkdir -v -p "${packagedefs.LIB_SYSTEMD}/system" + mkdir -v -p "${packagedefs.HTTP_DEV_DIV}" + mkdir -v -p "${packagedefs.LIB_SYSTEMD}/system" - cp -v -r "${SOURCE_PATH}/hmb-xr-5601-source/presto_xr/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + cp -v -r "${SOURCE_PATH}/hmb-xr-5601-source/presto_xr/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" - sudo chmod -v -R 755 "${packagedefs.DEBPATH}" - sudo chown -v -R root: "${packagedefs.DEBPATH}" + sudo chmod -v -R 755 "${packagedefs.DEBPATH}" + sudo chown -v -R root: "${packagedefs.DEBPATH}" - mkdir -v -p "${packageout}" - dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + mkdir -v -p "${packageout}" + dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" - dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" - dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" - - lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - """ + dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + + lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true + + sudo chown -v -R ${USER}: "${packagedefs.DEBPATH}" + """ + } + catch (Exception e) { + echo "Exception occurred" + sh """#!/bin/bash -v + echo "${password}" | sudo -S echo + + sudo chown -v -R ${USER}: "${packagedefs.DEBPATH}" + + exit 1 + """ + } } return this; \ No newline at end of file From d1f0c5254765420d292d051782a66402a9f1f98c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 12:43:42 +0100 Subject: [PATCH 515/558] HMBXR5601-827 - Change permissions after package build so that it can be cleaned on the next build --- packaging/debian.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index cef692f..4981c78 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -84,7 +84,7 @@ def buildPackage(String packageout, DebRootDefinition packagedefs, String passwo lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true - sudo chown -v -R ${USER}: "${packagedefs.DEBPATH}" + sudo chown -R ${USER}: "${packagedefs.DEBPATH}" """ } catch (Exception e) { @@ -92,7 +92,7 @@ def buildPackage(String packageout, DebRootDefinition packagedefs, String passwo sh """#!/bin/bash -v echo "${password}" | sudo -S echo - sudo chown -v -R ${USER}: "${packagedefs.DEBPATH}" + sudo chown -R ${USER}: "${packagedefs.DEBPATH}" exit 1 """ From fd69303adc548a52c1acb96ee04e0040e0993c26 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 12:47:12 +0100 Subject: [PATCH 516/558] HMBXR5601-827 - Change permissions after package build so that it can be cleaned on the next build --- packaging/debian.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 4981c78..1bd8a6e 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -77,11 +77,15 @@ def buildPackage(String packageout, DebRootDefinition packagedefs, String passwo sudo chown -v -R root: "${packagedefs.DEBPATH}" mkdir -v -p "${packageout}" + echo "Building package:" dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package:" dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package contents:" dpkg-deb --contents "${packageout}/${packagedefs.PACKAGE_NAME}" + echo "Showing package lintian output:" lintian --suppress-tags dir-or-file-in-opt "${packageout}/${packagedefs.PACKAGE_NAME}" || true sudo chown -R ${USER}: "${packagedefs.DEBPATH}" From b613549d453480a996767e74e667f35114b78a21 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 14:48:15 +0100 Subject: [PATCH 517/558] HMBXR5601-827 - Provide packages with unique filenames so that aptly stops complaining about identical files --- packaging/debian.groovy | 8 ++++++-- publish/package.groovy | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 1bd8a6e..7762951 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -24,11 +24,12 @@ class DebRootDefinition { String HTTP_DEV_DIV String LIB_SYSTEMD - DebRootDefinition(String rootpath, String packageprefix) { + DebRootDefinition(String rootpath, String packageprefix, String buildstamp) { ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix PACKAGE_NAME = packageprefix + "_armhf.deb" + UNIQUE_PACKAGE_NAME = packageprefix + "_" + buildstamp + "_armhf.deb" PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" @@ -56,7 +57,7 @@ def DebRootDefinition debDefinition(String rootpath, String packageprefix) { return new DebRootDefinition(rootpath, packageprefix) } -def buildPackage(String packageout, DebRootDefinition packagedefs, String password) { +def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { try { sh """#!/bin/bash # Exit if any command fails @@ -80,6 +81,9 @@ def buildPackage(String packageout, DebRootDefinition packagedefs, String passwo echo "Building package:" dpkg-deb -v --build "${packagedefs.DEBPATH}" "${packageout}/${packagedefs.PACKAGE_NAME}" + mkdir -v -p "${uniquepackageout}" + cp "${packageout}/${packagedefs.PACKAGE_NAME}" "${uniquepackageout}/${packagedefs.UNIQUE_PACKAGE_NAME}" + echo "Showing package:" dpkg-deb --show "${packageout}/${packagedefs.PACKAGE_NAME}" echo "Showing package contents:" diff --git a/publish/package.groovy b/publish/package.groovy index 32cc9eb..ea79cfd 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -50,6 +50,7 @@ pipeline { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") packagemeta = importer.artifactimport(meta, PACKAGE_CREATE_JOB, PACKAGE_CREATE_JOB_SELECTOR, PACKAGEDEST) PACKAGE_FOLDER = "${PACKAGEDEST}/package" + UNIQUE_PACKAGE_FOLDER = "${PACKAGEDEST}/uniquepackage" } } } @@ -97,9 +98,9 @@ pipeline { stage('Add/update package in repo') { steps { script { - echo "Package folder is ${PACKAGE_FOLDER}" + echo "Unique package folder is ${UNIQUE_PACKAGE_FOLDER}" sh """#!/bin/bash -v - aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${PACKAGE_FOLDER}" + aptly repo -config="${APTLY_CONFIG_FILE}" add "${REPO_NAME}" "${UNIQUE_PACKAGE_FOLDER}" if [ \$? -ne 0 ]; then exit 1; fi aptly repo -config="${APTLY_CONFIG_FILE}" show -with-packages "${REPO_NAME}" if [ \$? -ne 0 ]; then exit 1; fi From 48c62468007201c862390c4454fa7c39ffe3711c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 14:51:34 +0100 Subject: [PATCH 518/558] HMBXR5601-827 - Provide packages with unique filenames so that aptly stops complaining about identical files --- packaging/debian.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 7762951..0323cea 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -53,8 +53,8 @@ class DebRootDefinition { } } -def DebRootDefinition debDefinition(String rootpath, String packageprefix) { - return new DebRootDefinition(rootpath, packageprefix) +def DebRootDefinition debDefinition(String rootpath, String packageprefix, String buildstamp) { + return new DebRootDefinition(rootpath, packageprefix, buildstamp) } def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { From 8bc62d86f4899cf7171310d9ae12f4f66f5e47d0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Mon, 9 Sep 2019 14:53:07 +0100 Subject: [PATCH 519/558] HMBXR5601-827 - Provide packages with unique filenames so that aptly stops complaining about identical files --- packaging/debian.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 0323cea..0d6e7e4 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -3,6 +3,7 @@ class DebRootDefinition { String PACKAGE_PREFIX String PACKAGE_FOLDER_NAME String PACKAGE_NAME + String UNIQUE_PACKAGE_NAME String PACKAGE_SOURCE_FOLDER_NAME String DEBPATH From 9bf1c08f81e03beb74004d240c573505fd498720 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 10 Sep 2019 21:44:08 +0100 Subject: [PATCH 520/558] HMBXR5601-827 - Tar up export to preserve links --- compile/eclipsebuild/eclipseds5build.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 9ab8df3..63c8d45 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -81,7 +81,7 @@ def doExport(exportfolder, exportpathlist, partdest = "", build_variant = "") { echo "Copy \"${srcFileExpanded}\" to \"${destFolderExpanded}\"" if (isUnix()) { result = sh( script: """ - cp -r -v "${srcFileExpanded}" "${destFolderExpanded}" + cp -v --no-dereference --preserve=links -r "${srcFileExpanded}" "${destFolderExpanded}" if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true From 0b35442b42e83286cd74795edde980ee2e1cd58c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 11 Sep 2019 19:45:48 +0100 Subject: [PATCH 521/558] HMBXR5601-827 - Avoid XR specific path in Common_scripts --- packaging/debian.groovy | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 0d6e7e4..6c5240c 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -5,6 +5,7 @@ class DebRootDefinition { String PACKAGE_NAME String UNIQUE_PACKAGE_NAME + String PACKAGE_SOURCE_ROOT String PACKAGE_SOURCE_FOLDER_NAME String DEBPATH @@ -25,13 +26,14 @@ class DebRootDefinition { String HTTP_DEV_DIV String LIB_SYSTEMD - DebRootDefinition(String rootpath, String packageprefix, String buildstamp) { + DebRootDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp) { ROOTPATH = rootpath PACKAGE_PREFIX = packageprefix PACKAGE_FOLDER_NAME = packageprefix PACKAGE_NAME = packageprefix + "_armhf.deb" UNIQUE_PACKAGE_NAME = packageprefix + "_" + buildstamp + "_armhf.deb" + PACKAGE_SOURCE_ROOT = packagesourceroot PACKAGE_SOURCE_FOLDER_NAME = "${ROOTPATH}/${PACKAGE_PREFIX}" DEBPATH = "${ROOTPATH}/${PACKAGE_FOLDER_NAME}" @@ -54,8 +56,8 @@ class DebRootDefinition { } } -def DebRootDefinition debDefinition(String rootpath, String packageprefix, String buildstamp) { - return new DebRootDefinition(rootpath, packageprefix, buildstamp) +def DebRootDefinition debDefinition(String rootpath, String packagesourceroot, String packageprefix, String buildstamp) { + return new DebRootDefinition(rootpath, packagesourceroot, packageprefix, buildstamp) } def buildPackage(String packageout, String uniquepackageout, DebRootDefinition packagedefs, String password) { @@ -73,7 +75,7 @@ def buildPackage(String packageout, String uniquepackageout, DebRootDefinition p mkdir -v -p "${packagedefs.HTTP_DEV_DIV}" mkdir -v -p "${packagedefs.LIB_SYSTEMD}/system" - cp -v -r "${SOURCE_PATH}/hmb-xr-5601-source/presto_xr/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" + cp -v -r "${packagedefs.PACKAGE_SOURCE_ROOT}/${packagedefs.PACKAGE_SOURCE_FOLDER_NAME}"/* "${packagedefs.DEBPATH}" sudo chmod -v -R 755 "${packagedefs.DEBPATH}" sudo chown -v -R root: "${packagedefs.DEBPATH}" From d7ff5ae021b3a56bb9681f92cfb3d6b8d268948a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 11 Sep 2019 20:27:44 +0100 Subject: [PATCH 522/558] HMBXR5601-827 - Keep published repos of different products and branches seperate --- publish/package.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index ea79cfd..0f1ad9e 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -87,7 +87,7 @@ pipeline { script { sh """#!/bin/bash -v # This errors if the published repo exists already; Ignore it - aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" + aptly publish -config="${APTLY_CONFIG_FILE}" drop "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" aptly publish -config="${APTLY_CONFIG_FILE}" list if [ \$? -ne 0 ]; then exit 1; fi """ @@ -129,11 +129,11 @@ pipeline { steps { script { sh """#!/bin/bash -v - aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" + aptly publish -config="${APTLY_CONFIG_FILE}" -distribution="${DISTRIBUTION}" -component="${COMPONENT}" -gpg-key="${GPG_KEY_ID}" snapshot "${FULL_SNAPSHOT_NAME}" "${PUBLISH_PATH_PREFIX}" if [ \$? -ne 0 ]; then exit 1; fi aptly publish -config="${APTLY_CONFIG_FILE}" list if [ \$? -ne 0 ]; then exit 1; fi - aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" + aptly publish -config="${APTLY_CONFIG_FILE}" show "${DISTRIBUTION}" "${PUBLISH_PATH_PREFIX}" if [ \$? -ne 0 ]; then exit 1; fi """ } From 62e338e7f9da4e3ec89eecbe4f9b0484181b03e0 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 11 Sep 2019 23:30:06 +0100 Subject: [PATCH 523/558] HMBXR5601-827 - Had to reverse the changes of this commonly used script --- copy/logic.groovy | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index a5b134b..3f3f48d 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -24,12 +24,10 @@ pipeline { stage('Copying logic artifacts') { steps { script { - env.SG600_LOGICDEST = "${WORKSPACE}/sg600_logic" - env.KTB_LOGICDEST = "${WORKSPACE}/ktb_logic" + env.LOGICDEST = "${WORKSPACE}/logic" if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - importer.artifactimport(meta, KTB_LOGIC_ARTIFACT_PROJECT, KTB_LOGIC_ARTIFACT_BUILD_SELECTOR, KTB_LOGICDEST) - importer.artifactimport(meta, SG600_LOGIC_ARTIFACT_PROJECT, SG600_LOGIC_ARTIFACT_BUILD_SELECTOR, SG600_LOGICDEST) + importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) } } } @@ -39,7 +37,7 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf diff meta embedded_logic + rm -v -rf meta mv -v logic/* . """ archiveArtifacts "${artifactlist}" @@ -61,4 +59,4 @@ pipeline { echo 'Succeeded build' } } -} +} \ No newline at end of file From a67970c3105e97977d8178b046bda99a60bc51b5 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 12 Sep 2019 09:27:44 +0100 Subject: [PATCH 524/558] HMBXR5601-827 - Filter one branches packages from another --- publish/package.groovy | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index 0f1ad9e..9a77259 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -113,11 +113,16 @@ pipeline { script { echo "Snapshot name is ${SNAPSHOT_NAME}" FULL_SNAPSHOT_NAME = "${REPO_NAME}_${SNAPSHOT_NAME}" - echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME}" + PACKAGE_VERSION_PREFIX = packagemeta.buildstampinfo.buildstampprefix + echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" sh """#!/bin/bash -v # This errors if the snapshot exists already; Ignore it + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All" from repo "${REPO_NAME}" + if [ \$? -ne 0 ]; then exit 1; fi + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" - aptly snapshot -config="${APTLY_CONFIG_FILE}" create "${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All" ${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_PREFIX}.*)' if [ \$? -ne 0 ]; then exit 1; fi aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" if [ \$? -ne 0 ]; then exit 1; fi From f5a27e25ea078bd734b127285b707173263d9e2f Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 12 Sep 2019 09:30:28 +0100 Subject: [PATCH 525/558] HMBXR5601-827 - Filter one branches packages from another --- publish/package.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/package.groovy b/publish/package.groovy index 9a77259..9bd9824 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -122,7 +122,7 @@ pipeline { if [ \$? -ne 0 ]; then exit 1; fi aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" - aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All" ${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_PREFIX}.*)' + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_PREFIX}.*)' if [ \$? -ne 0 ]; then exit 1; fi aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" if [ \$? -ne 0 ]; then exit 1; fi From 2c3172d6dc8f6578696a3e0a91b6606f9d57407a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 12 Sep 2019 09:37:09 +0100 Subject: [PATCH 526/558] HMBXR5601-827 - Filter one branches packages from another --- publish/package.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/publish/package.groovy b/publish/package.groovy index 9bd9824..2d5fdcc 100644 --- a/publish/package.groovy +++ b/publish/package.groovy @@ -117,12 +117,12 @@ pipeline { echo "Full Snapshot name is ${FULL_SNAPSHOT_NAME} containing packages of versions beginning with: ${PACKAGE_VERSION_PREFIX}" sh """#!/bin/bash -v # This errors if the snapshot exists already; Ignore it - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All" - aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All" from repo "${REPO_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "All_${FULL_SNAPSHOT_NAME}" + aptly snapshot -config="${APTLY_CONFIG_FILE}" create "All_${FULL_SNAPSHOT_NAME}" from repo "${REPO_NAME}" if [ \$? -ne 0 ]; then exit 1; fi - aptly snapshot -config="${APTLY_CONFIG_FILE}" drop "${FULL_SNAPSHOT_NAME}" - aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_PREFIX}.*)' + aptly snapshot filter -config="/home/repos/gis-aptly-config/gis.conf" "All_${FULL_SNAPSHOT_NAME}" "${FULL_SNAPSHOT_NAME}" 'Version (~ .*${PACKAGE_VERSION_PREFIX}.*)' if [ \$? -ne 0 ]; then exit 1; fi aptly snapshot -config="${APTLY_CONFIG_FILE}" -with-packages show "${FULL_SNAPSHOT_NAME}" if [ \$? -ne 0 ]; then exit 1; fi From bc67f4dad128a81d3a081b71da6fdb8bec164a78 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 15 Sep 2019 14:17:49 +0100 Subject: [PATCH 527/558] GIS-1621 - Use script prompting to obtain publishing info from user --- publish/ftp_query.groovy | 18 +++++ publish/promptingpublish.groovy | 129 ++++++++++++++++++++++++++++++++ remoteaccess/ftp.groovy | 33 ++++++++ remoteaccess/hostedftp.groovy | 28 +++++++ versioncontrol/git.groovy | 6 ++ 5 files changed, 214 insertions(+) create mode 100644 publish/ftp_query.groovy create mode 100644 publish/promptingpublish.groovy create mode 100644 remoteaccess/ftp.groovy create mode 100644 remoteaccess/hostedftp.groovy diff --git a/publish/ftp_query.groovy b/publish/ftp_query.groovy new file mode 100644 index 0000000..02a4c2d --- /dev/null +++ b/publish/ftp_query.groovy @@ -0,0 +1,18 @@ +ftpremote = load("${autobuilder.commonscripts()}/remoteaccess/ftp.groovy") + +def getCustomerListFromFTP(FTPSiteInfo ftpsiteinfo) { + def List customerfilenames = new ArrayList() + def List customers = new ArrayList() + + FTPClient ftpclient = ftpremote.createFTPClient(ftpsiteinfo) + + customerfilenames = ftpclient.listNames() + for(customer in customerfilenames) { + def String converted = new String(customer.getBytes("ISO-8859-1"), "UTF-8") + customers.add(converted) + } + ftpclient.disconnect() + return customers +} + +return this; diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy new file mode 100644 index 0000000..deffec8 --- /dev/null +++ b/publish/promptingpublish.groovy @@ -0,0 +1,129 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + } + } + } + stage('Clean folders') { + steps { + script { + cleanfolders = [ + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Checkout release repo') { + steps { + script { + configurationhelper = autobuilder.configurationhelper() + + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "origin/master" + SOURCEDIR = "${WORKSPACE}/whole" + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + } + } + } + stage('Identify customer to publish to') { + steps { + script { + hostedftb = load("${autobuilder.commonscripts()}/remoteaccess/hostedftp.groovy") + def List customers = hostedftb.getCustomerList() + customer = input( + message: 'Please select customer to publish to', + ok: 'Ok', + parameters: [ + choice(name: 'customer', choices: customers, description: 'Which customer?') + ] + ) + echo "Chosen customer is: ${customer}" + } + } + } + stage('Identify software release to publish') { + steps { + script { + gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") + def List releases = gh.gettaglist(SOURCEDIR) + release = input( + message: 'Please select the software version', + ok: 'Ok', + parameters: [ + choice(name: 'release', choices: releases, description: 'Which release?') + ] + ) + echo "Chosen software version is: ${release}" + } + } + } + stage('Checkout release repo at selected software version') { + steps { + script { + SOURCEURL = RELEASE_REPO + SOURCEREVISION = "refs/tags/${release}" + SOURCEDIR = "${WORKSPACE}/whole" + configurationhelper.gitCheckoutAndRegister(meta, 'whole', SOURCEURL, SOURCEREVISION, SOURCEDIR, '', 0, false) + } + } + } + stage('Get release artifact meta') { + steps { + script { + def mfs = autobuilderscript.getmetafolderscript() + releasemeta = mfs.metafromfolder(SOURCEDIR) + } + } + } + stage('Publish components') { + steps { + script { + def publisher = load("${autobuilder.commonscripts()}/publish/do.groovy") + (summarystring, markunstable, unstablereason) = publisher.publishhierarchy(releasemeta, ITEMS_TO_PUBLISH, customer, release, MAX_META_DEPTH) + if (markunstable) { + env.unstablereason = unstablereason + currentBuild.result = 'UNSTABLE' + } + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts() + archiveArtifacts "summary/**,${artifactlist}" + } + } + } + stage('Summarise') { + steps { + script { + echo "Summary:\n${summarystring}\n" + if (env.unstablereason) { + echo "Instability reason: ${env.unstablereason}" + } + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} + diff --git a/remoteaccess/ftp.groovy b/remoteaccess/ftp.groovy new file mode 100644 index 0000000..750319f --- /dev/null +++ b/remoteaccess/ftp.groovy @@ -0,0 +1,33 @@ +@Grab(group='commons-net', module='commons-net', version='2.0') +import org.apache.commons.net.ftp.FTPClient + +class FTPSiteInfo { + String url + String username + String password + String workingdirectory + + FTPSiteInfo(String url, String username, String password, String workingdirectory) { + this.url = url + this.username = username + this.password = password + this.workingdirectory = workingdirectory + } +} + +FTPSiteInfo createFTPSiteInfo(String url, String username, String password, String workingdirectory) { + FTPSiteInfo ftpsiteinfo = new FTPSiteInfo(url, username, password, workingdirectory) + return ftpsiteinfo +} + +FTPClient createFTPClient(FTPSiteInfo ftpsiteinfo) { + FTPClient ftpclient = new FTPClient().with { + connect ftpsiteinfo.url + enterLocalPassiveMode() + login ftpsiteinfo.username, ftpsiteinfo.password + changeWorkingDirectory ftpsiteinfo.workingdirectory + } + return ftpclient +} + +return this; \ No newline at end of file diff --git a/remoteaccess/hostedftp.groovy b/remoteaccess/hostedftp.groovy new file mode 100644 index 0000000..fd81d6d --- /dev/null +++ b/remoteaccess/hostedftp.groovy @@ -0,0 +1,28 @@ +ftpremote = load("${autobuilder.commonscripts()}/remoteaccess/ftp.groovy") + +FTPSiteInfo getHostedFTPSiteInfo() { + FTPSiteInfo hostedftpsiteinfo = + ftpremote.createFTPSiteInfo("ftp.hostedftp.com", + "ftp@globalinkjetsystems.com", + "ramshorn42", + "/Customers/") + return hostedftpsiteinfo +} + +def getCustomerList() { + def List customerfilenames = new ArrayList() + def List customers = new ArrayList() + + FTPSiteInfo ftpsiteinfo = getHostedFTPSiteInfo() + FTPClient ftpclient = ftpremote.createFTPClient(ftpsiteinfo) + + customerfilenames = ftpclient.listNames() + for(customer in customerfilenames) { + def String converted = new String(customer.getBytes("ISO-8859-1"), "UTF-8") + customers.add(converted) + } + ftpclient.disconnect() + return customers +} + +return this; \ No newline at end of file diff --git a/versioncontrol/git.groovy b/versioncontrol/git.groovy index bb7c0ce..d24f3ff 100644 --- a/versioncontrol/git.groovy +++ b/versioncontrol/git.groovy @@ -25,6 +25,12 @@ String getGitInfo(String workingSetRootPath) { } } +List gettaglist(String workingSetRootPath) { + dir(workingSetRootPath) { + return clh.getCommandOutput('git tag --sort=-version:refname') + } +} + @NonCPS def ArrayList sparsecheckoutconfig(ArrayList paths) { ArrayList sparsecheckoutpaths = new ArrayList() From 2056fd98334dfadbf0870286341851f0e50bd974 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 17 Sep 2019 22:58:28 +0100 Subject: [PATCH 528/558] HMBFDHV-12 - Check that files are produced even though they won't be exported --- compile/eclipsebuild/eclipseds5build.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 63c8d45..888ad1e 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -40,7 +40,7 @@ def doCheck(checkpathlist, build_variant = "") { // Print out dependencies of the built file echo "Showing dependencies of ${srcFileExpanded}" result = sh( script: """ - readelf -d "${srcFileExpanded}" | grep NEEDED + readelf -d "${srcFileExpanded}" if [ \$? -ne 0 ]; then exit 1; fi """, returnStatus: true From 8be8e36ddf2cb9b70fc1284182e77446ff137b87 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sun, 22 Sep 2019 17:39:28 +0100 Subject: [PATCH 529/558] SMB-296 - Print out packaging commands --- packaging/debian.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/debian.groovy b/packaging/debian.groovy index 6c7e006..2ef1ba8 100644 --- a/packaging/debian.groovy +++ b/packaging/debian.groovy @@ -64,7 +64,7 @@ def buildPackage(String packageout, String uniquepackageout, DebRootDefinition p try { sh """#!/bin/bash # Exit if any command fails - set -e + set -xe echo "${password}" | sudo -S echo From 9282feb36dedac15d6cfb22cb2eb506a1b7dce49 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 11 Oct 2019 13:24:01 +0100 Subject: [PATCH 530/558] Add VS2017 installation configuration --- .../vs2017install/buildmachines.vsconfig | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tool configurations/vs2017install/buildmachines.vsconfig diff --git a/tool configurations/vs2017install/buildmachines.vsconfig b/tool configurations/vs2017install/buildmachines.vsconfig new file mode 100644 index 0000000..ea618a3 --- /dev/null +++ b/tool configurations/vs2017install/buildmachines.vsconfig @@ -0,0 +1,25 @@ +{ + "version": "1.0", + "components": [ + "Microsoft.VisualStudio.Workload.ManagedDesktop", + "Microsoft.VisualStudio.Workload.NativeDesktop", + "Microsoft.VisualStudio.Workload.NetWeb", + "Microsoft.VisualStudio.Workload.NetCoreTools", + "microsoft.net.componentgroup.targetingpacks.common", + "microsoft.componentgroup.blend", + "microsoft.visualstudio.component.entityframework", + "microsoft.visualstudio.component.debugger.justintime", + "microsoft.visualstudio.component.wcf.tooling", + "microsoft.visualstudio.component.vc.diagnostictools", + "microsoft.visualstudio.component.vc.cmake.project", + "microsoft.visualstudio.component.vc.testadapterforboosttest", + "microsoft.visualstudio.component.vc.testadapterforgoogletest", + "microsoft.visualstudio.component.vc.atlmfc", + "microsoft.visualstudio.component.appinsights.tools", + "microsoft.visualstudio.component.diagnostictools", + "microsoft.visualstudio.componentgroup.web.cloudtools", + "microsoft.visualstudio.component.aspnet45", + "microsoft.visualstudio.component.webdeploy", + "microsoft.visualstudio.component.nuget.buildtools" + ] +} \ No newline at end of file From b32a5116dd657f59b30ef079c6bc9c1056833796 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Oct 2019 16:18:19 +0100 Subject: [PATCH 531/558] GIS-1621 - Remove local diffing (superceded); Add as much git info to a meta object when it is not being provided - for old jobs not providing meta information --- configuration/configuration.groovy | 2 +- differences/do.groovy | 18 ------------------ import/importer.groovy | 4 ++-- meta/folder.groovy | 25 ++----------------------- 4 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 differences/do.groovy diff --git a/configuration/configuration.groovy b/configuration/configuration.groovy index b8c0b47..2a7f1ab 100644 --- a/configuration/configuration.groovy +++ b/configuration/configuration.groovy @@ -5,7 +5,7 @@ def registerConfigurationInfo(meta, String configurationName, String configurati } def registerPipelineScriptConfiguration(meta) { - registerConfigurationInfo(meta, 'pipeline script', '-', '-', gh.getGitInfo("${autobuilder.getautobuildpath()}")) + registerConfigurationInfo(meta, 'pipeline script', '-', '-', gh.getGitInfo(autobuilder.getautobuildpath())) } def gitCheckoutAndRegister(meta, String tool, String gitURL, String revision, String targetDir, String repoRoot = "", int cloneDepth = 1, boolean shallowClone = true, String filter = '') { diff --git a/differences/do.groovy b/differences/do.groovy deleted file mode 100644 index 32f18b5..0000000 --- a/differences/do.groovy +++ /dev/null @@ -1,18 +0,0 @@ - -def differences(meta, String sourcerevision, String previoussourcerevision, String sourcepath) { - def String cs = autobuilder.commonscripts() - def String difffile = meta.difffilepath() - def String jirakey = meta.productinfo.jirakey - if (isUnix()) { - sh """ - "${PYTHON}" -u "${cs}/differences/show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" - """ - } - else { - bat """ - "${PYTHON}" -u "${cs}\\differences\\show_release_differences.py" -g "${sourcepath}" -s "${sourcerevision}" -p "${previoussourcerevision}" -j "${jirakey}" -o "opposite notcommon" -f "${difffile}" - """ - } -} - -return this; diff --git a/import/importer.groovy b/import/importer.groovy index ba5eb0b..78c0d62 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -51,7 +51,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) relatedimportinfo = importedmeta.getassociated() echo "Cloning associated repos" @@ -90,7 +90,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) - def importedmeta = meta.importrelated(destpath) + def importedmeta = meta.importrelated(destpath, rooturl, rootrevision, gh.getGitInfo(destpath)) gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" diff --git a/meta/folder.groovy b/meta/folder.groovy index 6967def..6371f38 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -429,11 +429,8 @@ class PublishableInfo extends GISJsonSerialised { class MetaInfo extends GISJsonSerialised { private static String FORMAT = "0.1" private static String METAFOLDERNAME = "meta" - private static String DIFFFOLDERNAME = "diff" private static String METAFILENAME = "info.json" - private static String METADIFFFILENAME = "differences.txt" private static String METAARTIFACTS = "${METAFOLDERNAME}/**" - private static String DIFFARTIFACTS = "${DIFFFOLDERNAME}/**" class RelatedMetaInfo extends GISJsonSerialised { String offsetpath @@ -607,11 +604,7 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME + "/" + METAFILENAME } - static String difffilepath() { - return DIFFFOLDERNAME + "/" + METADIFFFILENAME - } - - MetaInfo importrelated(String destpath) { + MetaInfo importrelated(String destpath, String givenRepo = 'Unknown', String givenRevision = 'Unknown', String givenSHA = 'Unknown') { MetaInfo mi = new MetaInfo(script) def String adjustedpath = relativize(destpath) @@ -620,22 +613,12 @@ class MetaInfo extends GISJsonSerialised { mi.deserialiseJson(importedmetafile) } else { script.echo("Could not deserialise meta at file: ${importedmetafile} (destpath: ${destpath})- Does not exist") + mi.addConfigurationInfo('', givenRepo, givenRevision, givenSHA) } RelatedMetaInfo rmi = new RelatedMetaInfo(script, adjustedpath, mi) associatedmeta.add(rmi) - def importedmetadifffile = adjustedpath + "/" + difffilepath() - if (script.fileExists(importedmetadifffile)) { - def String newcontents = script.readFile(file: importedmetadifffile) - def String currentcontents = "" - if (script.fileExists(difffilepath())) { - currentcontents = script.readFile(file: difffilepath()) - } - - script.writeFile(file: difffilepath(), text: currentcontents + "\n" + mi.productinfo.productname + " " + mi.productinfo.subcomponentname + ":\n" + newcontents) - } - return mi } @@ -660,10 +643,6 @@ class MetaInfo extends GISJsonSerialised { return METAFOLDERNAME } - static String difffolder() { - return DIFFFOLDERNAME - } - String generateartifacts(ArrayList releasedfromhere = new ArrayList(), // These items are recorded in meta as released from here ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" From f6a55441cf4edca1b789332df255a59ae3979843 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Oct 2019 16:20:57 +0100 Subject: [PATCH 532/558] GIS-1621 - Remove local diffing (superceded); Add as much git info to a meta object when it is not being provided - for old jobs not providing meta information --- meta/folder.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 6371f38..2e142a2 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -756,10 +756,6 @@ MetaInfo metareset(script) { deleteDir() } - dir(mi.difffolder()) { - deleteDir() - } - return mi } From a13f11435b7dec72fe8a98a2838b6995ec890f5a Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 25 Oct 2019 16:59:55 +0100 Subject: [PATCH 533/558] GIS-1621 - Remove local diffing (superceded); Add as much git info to a meta object when it is not being provided - for old jobs not providing meta information --- meta/folder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 2e142a2..e5f34f0 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -647,7 +647,7 @@ class MetaInfo extends GISJsonSerialised { ArrayList releasedfromsubmeta = new ArrayList(), // These items are not recorded in meta as released from here String metafilepath ="${METAFOLDERNAME}/${METAFILENAME}" ) { - def String artifactlist = "${METAARTIFACTS},${DIFFARTIFACTS}" + def String artifactlist = METAARTIFACTS releaseinfo.addReleaseItem(METAARTIFACTS) for (releaseitem in releasedfromhere) { artifactlist += "," + releaseitem From d0ed9310fc8502e8153cc8277341f734368c620c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:11:10 +0000 Subject: [PATCH 534/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 16 +++++++++++++++- publish/publishitems.groovy | 2 +- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index ced73a2..670501d 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -53,7 +53,21 @@ pipeline { steps { script { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") - def releases = gh.gettaglist(SOURCEDIR) + def allreleases = gh.gettaglist(SOURCEDIR) + showall = SHOWALL ?: 'false' + def releases + if (showall) { + releases = allreleases + } else { + for (release in allreleases) { + if (!release.toLowerCase().startsWith("superceded_") && + !release.toLowerCase().startsWith("erroneous_") && + !release.toLowerCase().startsWith("faulty_") && + !release.toLowerCase().startsWith("inmainline_")) { + releases.append(release) + } + } + } timeout(time: 10, unit: 'MINUTES') { release = input( message: 'Please select the software version', diff --git a/publish/publishitems.groovy b/publish/publishitems.groovy index d84123d..677970e 100644 --- a/publish/publishitems.groovy +++ b/publish/publishitems.groovy @@ -124,4 +124,4 @@ PublishableItemsSelection createItemSelector( return pis } -return this; \ No newline at end of file +return this; From 1f55e79acfa5989183c409e7e6d8f60ee96ed57b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:22:23 +0000 Subject: [PATCH 535/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 670501d..fb312f0 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -63,6 +63,7 @@ pipeline { if (!release.toLowerCase().startsWith("superceded_") && !release.toLowerCase().startsWith("erroneous_") && !release.toLowerCase().startsWith("faulty_") && + !release.toLowerCase().startsWith("ignore_") && !release.toLowerCase().startsWith("inmainline_")) { releases.append(release) } From fc3d611c844ed2219061aae4d2fd4b5cde7f2398 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:45:09 +0000 Subject: [PATCH 536/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index fb312f0..6950754 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -55,11 +55,13 @@ pipeline { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) showall = SHOWALL ?: 'false' + echo("showall: ${showall} ${SHOWALL}") def releases if (showall) { releases = allreleases } else { for (release in allreleases) { + echo("Evaluating: |${release}|") if (!release.toLowerCase().startsWith("superceded_") && !release.toLowerCase().startsWith("erroneous_") && !release.toLowerCase().startsWith("faulty_") && From 6ce20315af1d097c90e5bbfbfe51c6e1192c1520 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:47:59 +0000 Subject: [PATCH 537/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 6950754..80dc633 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -54,7 +54,7 @@ pipeline { script { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) - showall = SHOWALL ?: 'false' + boolean showall = (SHOWALL == 'true') echo("showall: ${showall} ${SHOWALL}") def releases if (showall) { From 1bd181cd0b3f0ca44d0d61f67c7b73c08e335d5b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:50:30 +0000 Subject: [PATCH 538/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 80dc633..bf95bb2 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -55,8 +55,7 @@ pipeline { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) boolean showall = (SHOWALL == 'true') - echo("showall: ${showall} ${SHOWALL}") - def releases + def releases = new ArrayList() if (showall) { releases = allreleases } else { From 3e27e84050ddd73e8b852970a23c4b6a034962da Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:53:52 +0000 Subject: [PATCH 539/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index bf95bb2..d8d5303 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -66,7 +66,7 @@ pipeline { !release.toLowerCase().startsWith("faulty_") && !release.toLowerCase().startsWith("ignore_") && !release.toLowerCase().startsWith("inmainline_")) { - releases.append(release) + releases.add(release) } } } From d5ab89e81edbc4d984bebcc026ec7a1b67d1b703 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 09:55:55 +0000 Subject: [PATCH 540/558] GIS-1621 - Be selective with releases - don't show those with tags beginning with erroneous_ superceded_ faulty_ inmainline_ unless SHOWALL parameter is true --- publish/promptingpublish.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index d8d5303..f2372cd 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -60,7 +60,6 @@ pipeline { releases = allreleases } else { for (release in allreleases) { - echo("Evaluating: |${release}|") if (!release.toLowerCase().startsWith("superceded_") && !release.toLowerCase().startsWith("erroneous_") && !release.toLowerCase().startsWith("faulty_") && From 15f4c4acd3800602327f91f082cb08653e6b0c08 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 7 Nov 2019 17:39:56 +0000 Subject: [PATCH 541/558] GIS-1621 - Add some output to debug why configuration information is not being added to imported logic meta --- import/importer.groovy | 4 ++-- meta/folder.groovy | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/import/importer.groovy b/import/importer.groovy index 78c0d62..25b2614 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -85,7 +85,7 @@ def hierarchyartifactimportfromrepo(meta, String repo, String revision, String d def artifactimportfromrepo(meta, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false) { def rooturl = repo.trim() def rootrevision = revision.trim() - echo "Cloning root release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" + echo "Cloning release repo: ${rooturl}, revision: ${rootrevision} at folder: ${destpath}" def ArrayList sparseitemlist = gh.convertSparseItemsString(filter) gh.gitCheckoutAlt(rooturl, rootrevision, destpath, "", 1, true, sparseitemlist) @@ -94,7 +94,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, gh.stripgithistory(destpath) def metafilepath = "${destpath}/${meta.metafilepath()}" - echo "Marking root meta as released and rewriting meta file: ${metafilepath}" + echo "Marking meta as released and rewriting meta file: ${metafilepath}" rewritemetafileasreleased(importedmeta, metafilepath) if (incrementversionnumber) { diff --git a/meta/folder.groovy b/meta/folder.groovy index e5f34f0..40f03aa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -511,6 +511,7 @@ class MetaInfo extends GISJsonSerialised { void addConfigurationInfo(String configurationname, String configurationURL, String configurationversion, String configurationSHA) { def ConfigurationInfo ci = new ConfigurationInfo(script, configurationname, configurationURL, configurationversion, configurationSHA) + script.echo("Adding configuration info: name: ${configurationname} url: ${configurationURL} version: ${configurationversion} sha: ${configurationSHA}") configurationinfo.add(ci) } From a006c13b6280e72626dad107020906361787383e Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 8 Nov 2019 17:44:18 +0000 Subject: [PATCH 542/558] GIS-1621 - Debug release filtering --- publish/promptingpublish.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index f2372cd..6f0a6a5 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -54,17 +54,20 @@ pipeline { script { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) - boolean showall = (SHOWALL == 'true') + boolean filterredundantreleases = (FILTER_REDUNDANT_RELEASES == 'true') def releases = new ArrayList() - if (showall) { + if (!filterredundantreleases) { + echo("Showing all releases") releases = allreleases } else { + echo("Showing filtered releases") for (release in allreleases) { if (!release.toLowerCase().startsWith("superceded_") && !release.toLowerCase().startsWith("erroneous_") && !release.toLowerCase().startsWith("faulty_") && !release.toLowerCase().startsWith("ignore_") && !release.toLowerCase().startsWith("inmainline_")) { + echo("Showing release: ${release}") releases.add(release) } } From 29bd0eafc6d2df94d8d5d9e6cd9a372bbb5d11af Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 9 Nov 2019 00:41:41 +0000 Subject: [PATCH 543/558] GIS-1621 - Show origin just before pushing tags --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index 1b21bcf..ee1bdf5 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -274,6 +274,7 @@ def releasegitpush(String dryrunpush ) { } def sourcegitpush(String dryrunpush ) { + sh("git remote -v") if (dryrunpush == "true") { sh("git push -n --tags") summaryappend(",Src-Dry-Run-Push") From 4e45329fe1c127ee158757295f45f97823bdadae Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 9 Nov 2019 07:58:41 +0000 Subject: [PATCH 544/558] GIS-1621 - Show origin just before pushing tags --- release/do.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/release/do.groovy b/release/do.groovy index ee1bdf5..a9f0a3e 100644 --- a/release/do.groovy +++ b/release/do.groovy @@ -262,6 +262,7 @@ def sourcegittags(String buildsha, List taglist) { } def releasegitpush(String dryrunpush ) { + sh("git remote -v") if (dryrunpush == "true") { sh("git push -n --tags origin master") summaryappend(",Dry-Run-Push") From da6b830e8cc068a105ac89e3cb5905ee16fc16c4 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 9 Nov 2019 08:03:23 +0000 Subject: [PATCH 545/558] GIS-1621 - Debug release filtering --- publish/promptingpublish.groovy | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 6f0a6a5..29e9bfd 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -62,11 +62,11 @@ pipeline { } else { echo("Showing filtered releases") for (release in allreleases) { - if (!release.toLowerCase().startsWith("superceded_") && - !release.toLowerCase().startsWith("erroneous_") && - !release.toLowerCase().startsWith("faulty_") && - !release.toLowerCase().startsWith("ignore_") && - !release.toLowerCase().startsWith("inmainline_")) { + if (!release.toLowerCase().startsWith("superceded") && + !release.toLowerCase().startsWith("erroneous") && + !release.toLowerCase().startsWith("faulty") && + !release.toLowerCase().startsWith("ignore") && + !release.toLowerCase().startsWith("inmainline")) { echo("Showing release: ${release}") releases.add(release) } From 6b5490069869ca03c4284a2eaa1e39b7574a9558 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 9 Nov 2019 08:08:42 +0000 Subject: [PATCH 546/558] GIS-1621 - Debug release filtering --- publish/promptingpublish.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 29e9bfd..80305ab 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -69,6 +69,8 @@ pipeline { !release.toLowerCase().startsWith("inmainline")) { echo("Showing release: ${release}") releases.add(release) + } else { + echo("Not showing release: ${release}") } } } From f64c71e0cd54074f5f941a465619712ed52e547c Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 9 Nov 2019 08:53:13 +0000 Subject: [PATCH 547/558] GIS-1621 - Remove publish filter debug log --- publish/promptingpublish.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index 80305ab..c0e4837 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -54,9 +54,9 @@ pipeline { script { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) - boolean filterredundantreleases = (FILTER_REDUNDANT_RELEASES == 'true') + boolean filterredundantreleases = def releases = new ArrayList() - if (!filterredundantreleases) { + if (FILTER_REDUNDANT_RELEASES == 'false') { echo("Showing all releases") releases = allreleases } else { @@ -67,10 +67,10 @@ pipeline { !release.toLowerCase().startsWith("faulty") && !release.toLowerCase().startsWith("ignore") && !release.toLowerCase().startsWith("inmainline")) { - echo("Showing release: ${release}") + //echo("Showing release: ${release}") releases.add(release) } else { - echo("Not showing release: ${release}") + //echo("Not showing release: ${release}") } } } From 0958de510814aa41340c8edf15d37d03e3f26ed3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Sat, 9 Nov 2019 09:16:06 +0000 Subject: [PATCH 548/558] GIS-1621 - Remove publish filter debug log --- publish/promptingpublish.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/publish/promptingpublish.groovy b/publish/promptingpublish.groovy index c0e4837..9f2d7dc 100644 --- a/publish/promptingpublish.groovy +++ b/publish/promptingpublish.groovy @@ -54,7 +54,6 @@ pipeline { script { gh = load("${autobuilder.commonscripts()}/versioncontrol/git.groovy") def allreleases = gh.gettaglist(SOURCEDIR) - boolean filterredundantreleases = def releases = new ArrayList() if (FILTER_REDUNDANT_RELEASES == 'false') { echo("Showing all releases") From 6a2df6c31c33851f13d694425f2c4f97f66cb5aa Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 28 Nov 2019 09:11:52 +0000 Subject: [PATCH 549/558] Increase build timeout - which is now tripping on longer builds --- compile/eclipsebuild/eclipseds5build.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile/eclipsebuild/eclipseds5build.groovy b/compile/eclipsebuild/eclipseds5build.groovy index 888ad1e..3e58e4f 100644 --- a/compile/eclipsebuild/eclipseds5build.groovy +++ b/compile/eclipsebuild/eclipseds5build.groovy @@ -156,7 +156,7 @@ def doBuildAndExportProject(String ds5folder, String toolchainfolder, String ds5 doDisplayLeftovers() doRegisterToolchain(ds5folder, toolchainfolder) echo "Build command: ${ds5shellcommand} \'${buildcommand} -no-indexer -build \"${project}/${build_variant}\"\'" - timeout(time: 10, unit: 'MINUTES') { + timeout(time: 20, unit: 'MINUTES') { if (isUnix()) { sh """ echo Start time: `date` From c38b0dc2d061d837aa2164d91e3c9144a8c1a523 Mon Sep 17 00:00:00 2001 From: "george.linsdell" Date: Mon, 2 Dec 2019 15:40:29 +0000 Subject: [PATCH 550/558] Checkout python and use it to run the collation step --- meta/folder.groovy | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/meta/folder.groovy b/meta/folder.groovy index 40f03aa..474e756 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,6 +19,26 @@ abstract class GISJsonSerialised implements Serializable { } } +class TestInfo extends GISJsonSerialised{ + String buildjobname + int buildnumber + + TestInfo(script) { + super(script) + script.echo("Initializing Test Info Object") + } + + void setInfoFromFile(String filepath){ + script.echo("TestInfo: Deserialising JSON at file: ${filepath}") + def rawmeta = script.readJSON(file: filepath) + deserialiseJson(rawmeta) + } + + void deserialiseJson(rawmeta) { + TestInfo = rawmeta['TestInfo'] + } +} + class ProductInfo extends GISJsonSerialised { String productname String subcomponentname From 43642b3b58fecb1946b0b266cfecad8da4543a6e Mon Sep 17 00:00:00 2001 From: "george.linsdell" Date: Mon, 2 Dec 2019 15:43:24 +0000 Subject: [PATCH 551/558] revert prmature change --- meta/folder.groovy | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/meta/folder.groovy b/meta/folder.groovy index 474e756..40f03aa 100644 --- a/meta/folder.groovy +++ b/meta/folder.groovy @@ -19,26 +19,6 @@ abstract class GISJsonSerialised implements Serializable { } } -class TestInfo extends GISJsonSerialised{ - String buildjobname - int buildnumber - - TestInfo(script) { - super(script) - script.echo("Initializing Test Info Object") - } - - void setInfoFromFile(String filepath){ - script.echo("TestInfo: Deserialising JSON at file: ${filepath}") - def rawmeta = script.readJSON(file: filepath) - deserialiseJson(rawmeta) - } - - void deserialiseJson(rawmeta) { - TestInfo = rawmeta['TestInfo'] - } -} - class ProductInfo extends GISJsonSerialised { String productname String subcomponentname From 24c4df35abda438d2da67cc95029d484ed367a08 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Wed, 4 Dec 2019 22:51:52 +0000 Subject: [PATCH 552/558] HMBXR5601-186 - Replace original init image generation with latest; Lose redundant partitions from image --- versionnumbering/generate.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/versionnumbering/generate.groovy b/versionnumbering/generate.groovy index 2e4f60d..de60b66 100644 --- a/versionnumbering/generate.groovy +++ b/versionnumbering/generate.groovy @@ -159,4 +159,11 @@ String generate(meta, String sourcerevision, String sourcepath, String basebuild return generate(meta, sourcerevision, sourcepath, bbn) } +String copyfrom(meta, othermeta) { + meta.gitinfo = othermeta.gitinfo + meta.buildstampinfo = othermeta.buildstampinfo + + return meta.buildstampinfo.buildstamp +} + return this; \ No newline at end of file From 7bc58a493dc7afc2da3ad6f3b450c52dce3e944b Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 6 Dec 2019 17:48:05 +0000 Subject: [PATCH 553/558] GIS-1621 - Add a title to each report --- differences/dometahierarchy.groovy | 54 ++++++++++++++++++------------ 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index 11b3733..e4095af 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -405,9 +405,9 @@ class MergeReportRecord { } } -def mergereport(String reportsubfolder, LinkedHashMap merges, boolean iscomponent = true) { +def mergereport(String reportsubfolder, String title, LinkedHashMap merges, boolean iscomponent = true) { dir(reportsubfolder) { - def String mergelist = "Merge list:\n" + def String mergelist = "${title}\nMerge list:\n" def mergessorted = merges.sort() for (amerge in mergessorted) { def MergeReportRecord m = amerge.value @@ -439,9 +439,9 @@ class RevertsReportRecord { } } -def revertsreport(String reportsubfolder, LinkedHashMap reverts, boolean iscomponent = true) { +def revertsreport(String reportsubfolder, String title, LinkedHashMap reverts, boolean iscomponent = true) { dir(reportsubfolder) { - def String revertlist = "Revert list:\n" + def String revertlist = "${title}\nRevert list:\n" def revertssorted = reverts.sort() for (arevert in revertssorted) { def RevertsReportRecord r = arevert.value @@ -470,9 +470,9 @@ class TicketedReportRecord { } } -def ticketedreport(String reportsubfolder, LinkedHashMap ticketed, boolean iscomponent = true) { +def ticketedreport(String reportsubfolder, String title, LinkedHashMap ticketed, boolean iscomponent = true) { dir(reportsubfolder) { - def String ticketedlist = "Ticketed list:\n" + def String ticketedlist = "${title}\nTicketed list:\n" def ticketedsorted = ticketed.sort() for (aticketed in ticketedsorted) { def TicketedReportRecord r = aticketed.value @@ -499,9 +499,9 @@ class UnticketedReportRecord { } } -def unticketedreport(String reportsubfolder, LinkedHashMap unticketed, boolean iscomponent = true) { +def unticketedreport(String reportsubfolder, String title, LinkedHashMap unticketed, boolean iscomponent = true) { dir(reportsubfolder) { - def String unticketedlist = "Unticketed list:\n" + def String unticketedlist = "${title}\nUnticketed list:\n" def unticketedsorted = unticketed.sort() for (anunticketed in unticketedsorted) { UnticketedReportRecord u = anunticketed.value @@ -523,9 +523,9 @@ class TicketsReportRecord { } } -def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { +def ticketsreport(String reportsubfolder, String title, tickets, boolean iscomponent = true) { dir(reportsubfolder) { - String ticketlist = "Ticket list:\n" + String ticketlist = "${title}\nTicket list:\n" ticketssorted = tickets.sort() for (ticketnumber in ticketssorted) { TicketsReportRecord t = ticketnumber.value @@ -535,9 +535,9 @@ def ticketsreport(String reportsubfolder, tickets, boolean iscomponent = true) { } } -def differencerecordsreport(String reportsubfolder, differencerecords, boolean iscomponent = true) { +def differencerecordsreport(String reportsubfolder, String title, differencerecords, boolean iscomponent = true) { dir(reportsubfolder) { - String recordlist = "" + String recordlist = "${title}\n" for (dr in differencerecords) { recordlist += dr.dump() + "\n\n" } @@ -545,7 +545,7 @@ def differencerecordsreport(String reportsubfolder, differencerecords, boolean i } } -def boolean report(meta, differencerecords, boolean iscomponent = true) { +def boolean report(meta, title, differencerecords, boolean iscomponent = true) { def Map ticketed = [:] def Map merges = [:] def Map reverts = [:] @@ -664,12 +664,24 @@ def boolean report(meta, differencerecords, boolean iscomponent = true) { } else { reportsubfolder = "${reportfolder}/Combined" } - mergereport(reportsubfolder, merges, iscomponent) - revertsreport(reportsubfolder, reverts, iscomponent) - ticketedreport(reportsubfolder, ticketed, iscomponent) - unticketedreport(reportsubfolder, unticketed, iscomponent) - ticketsreport(reportsubfolder, tickets, iscomponent) - differencerecordsreport(reportsubfolder, differencerecords, iscomponent) + mergereport(reportsubfolder, title, merges, iscomponent) + revertsreport(reportsubfolder, title, reverts, iscomponent) + ticketedreport(reportsubfolder, title, ticketed, iscomponent) + unticketedreport(reportsubfolder, title, unticketed, iscomponent) + ticketsreport(reportsubfolder, title, tickets, iscomponent) + differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) +} + +String def title(earlymeta, latermeta) { + return """ + Differences in: + Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} + between: + then: + Build date: ${earlymeta.jenkinsbuildinfo.builddate} Build stamp: ${earlymeta.buildstampinfo.buildstamp} Git revision: ${earlymeta.gitinfo.gitrevision} + now: + Build date: ${latermeta.jenkinsbuildinfo.builddate} Build stamp: ${latermeta.buildstampinfo.buildstamp} Git revision: ${latermeta.gitinfo.gitrevision} + """ } def compare(earlymetaitems, latermetaitems, key) { @@ -690,7 +702,7 @@ def compare(earlymetaitems, latermetaitems, key) { (result, differencerecords) = getdifferencerecords(earlymeta, latermeta, uniquefolderforsourcerepo) summaryappend(" Items: ${differencerecords.size()} ") - report(earlymeta, differencerecords) + report(earlymeta, title(earlymeta, latermeta), differencerecords) } else { summaryappend(" No source repo ") } @@ -737,7 +749,7 @@ def comparehierarchy(meta, String earlymetafolder, String latermetafolder, Strin summaryappend("\n") } } - report(meta, differencerecords, false) + report(meta, title(earlymeta, latermeta), differencerecords, false) return [summarystring, markunstable, unstablereason] } From d2a31e9baffe4e4d9c3493f7bec2e30d6b97da18 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 6 Dec 2019 17:50:17 +0000 Subject: [PATCH 554/558] GIS-1621 - Add a title to each report --- differences/dometahierarchy.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/differences/dometahierarchy.groovy b/differences/dometahierarchy.groovy index e4095af..2e94820 100644 --- a/differences/dometahierarchy.groovy +++ b/differences/dometahierarchy.groovy @@ -672,7 +672,7 @@ def boolean report(meta, title, differencerecords, boolean iscomponent = true) { differencerecordsreport(reportsubfolder, title, differencerecords, iscomponent) } -String def title(earlymeta, latermeta) { +def title(earlymeta, latermeta) { return """ Differences in: Product: ${earlymeta.productinfo.productname} Subcomponent: ${earlymeta.productinfo.subcomponentname} From 958ad765d03041dbe18bf4e80b1697bdeddb7aa3 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Thu, 12 Dec 2019 11:32:56 +0000 Subject: [PATCH 555/558] GIS-1621 - Handle FD1200 embedded software building --- import/importer.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/import/importer.groovy b/import/importer.groovy index 25b2614..a8b64f5 100644 --- a/import/importer.groovy +++ b/import/importer.groovy @@ -106,6 +106,7 @@ def artifactimportfromrepo(meta, String repo, String revision, String destpath, def artifactimport(meta, String source, String jenkinsjob, buildnumberorselector, String repo, String revision, String destpath, boolean incrementversionnumber = true, String filter = '', String excludes = '', boolean fingerprint = false, boolean nested = false) { switch (source) { case "Jenkins job artifact": + case "Jenkins job artefact": return artifactimport(meta, jenkinsjob, buildnumberorselector, destpath, incrementversionnumber, filter, excludes, fingerprint) case "Release repository": if (nested) { From 21c59682a1976ee78a41222bdbcbdd27be736866 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Fri, 13 Dec 2019 14:32:32 +0000 Subject: [PATCH 556/558] GIS-1621 - Make an ESW copier job script --- copy/esw.groovy | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 copy/esw.groovy diff --git a/copy/esw.groovy b/copy/esw.groovy new file mode 100644 index 0000000..2a63171 --- /dev/null +++ b/copy/esw.groovy @@ -0,0 +1,63 @@ +pipeline { + agent { label "${AGENT_LABEL}" } + stages { + stage('Initial setup') { + steps { + script { + meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") + } + } + } + stage('Clean exports') { + steps { + script { + env.ESW = 'esw' + env.ESWDEST = "${WORKSPACE}/${ESW}" + env.ESWEXPORT = 'embedded_software' + cleanfolders = [ + ESWDEST, + ESWEXPORT, + ] + cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") + cleaner.clean(cleanfolders) + } + } + } + stage('Copying embedded software artifacts') { + steps { + script { + importer.artifactimport(meta, ESW_ARTIFACT_LOCATION, ESW_ARTIFACT_BUILD_JOB, ESW_ARTIFACT_BUILD_SELECTOR, ESW_ARTIFACT_RELEASE_REPO, ESW_ARTIFACT_RELEASE_REVISION, ESWDEST) + } + } + } + stage('Archiving artifacts') { + steps { + script { + def artifactlist = meta.generateartifacts(["${ESWEXPORT}/**"]) + sh """ + rm -v -rf diff + rm -v -rf meta + mv -v "${ESW}"/* . + """ + archiveArtifacts "${artifactlist}" + } + } + } + } + post { + always { + echo 'Post-build' + } + failure { + echo 'Failed build' + } + unstable { + echo 'Unstable build' + } + success { + echo 'Succeeded build' + } + } +} \ No newline at end of file From 2241c0c982e1b9a5a36fbd2660b8f9de99bc4631 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 17 Dec 2019 18:38:57 +0000 Subject: [PATCH 557/558] GIS-1621 - Allow logic copies from repositories --- copy/logic.groovy | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 4b3d521..8c82d02 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -5,16 +5,18 @@ pipeline { steps { script { meta = load("${WORKSPACE}/autobuild/Common_scripts/autobuilder.groovy").startnonbuild(this) + + importer = load("${autobuilder.commonscripts()}/import/importer.groovy") } } } stage('Clean exports') { steps { script { + env.LOGICDEST = "${WORKSPACE}/logic" cleanfolders = [ - "${WORKSPACE}/logic", + LOGICDEST, "${WORKSPACE}/embedded_logic", - "${WORKSPACE}/diff", // Some old artifacts have this folder ] cleaner = load("${autobuilder.commonscripts()}/clean/clean.groovy") cleaner.clean(cleanfolders) @@ -24,11 +26,7 @@ pipeline { stage('Copying logic artifacts') { steps { script { - env.LOGICDEST = "${WORKSPACE}/logic" - if (LOGIC_ARTIFACT_LOCATION == "Jenkins job artifact") { - importer = load("${autobuilder.commonscripts()}/import/importer.groovy") - importer.artifactimport(meta, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGICDEST) - } + importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC _RELEASE_REVISION, LOGICDEST) } } } @@ -37,7 +35,6 @@ pipeline { script { def artifactlist = meta.generateartifacts(["embedded_logic/**"]) sh """ - rm -v -rf diff rm -v -rf meta mv -v "logic"/* . """ From d0f9244af4c83124b338a54c8ac36ece48973290 Mon Sep 17 00:00:00 2001 From: Dan Hawkins Date: Tue, 17 Dec 2019 18:43:14 +0000 Subject: [PATCH 558/558] GIS-1621 - Allow logic copies from repositories --- copy/logic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copy/logic.groovy b/copy/logic.groovy index 8c82d02..4d5291c 100644 --- a/copy/logic.groovy +++ b/copy/logic.groovy @@ -26,7 +26,7 @@ pipeline { stage('Copying logic artifacts') { steps { script { - importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC _RELEASE_REVISION, LOGICDEST) + importer.artifactimport(meta, LOGIC_ARTIFACT_LOCATION, LOGIC_ARTIFACT_PROJECT, LOGIC_ARTIFACT_BUILD_SELECTOR, LOGIC_RELEASE_REPO, LOGIC_RELEASE_REVISION, LOGICDEST) } } }