diff --git a/.jenkins_builds b/.jenkins_builds index 1336ecd0..d013ddb5 100644 --- a/.jenkins_builds +++ b/.jenkins_builds @@ -5,68 +5,88 @@ pipeline { environment { DEVELOPER_PRIVATE_KEY = "/.ci/repertory/cert/developer.priv" - DEVELOPER_PUBLIC_KEY = "/.ci/repertory/cert/developer.pub" + DEVELOPER_PUBLIC_KEY = "/.ci/repertory/cert/developer.pub" PROJECT_TEST_CONFIG_DIR = "/.ci/repertory/test" } options { disableConcurrentBuilds() - retry(2) + skipDefaultCheckout() + timestamps() } stages { - stage('linux_x86_64') { + stage('Build • Test • Deliver') { agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/make_unix.sh' + stages { + stage('Checkout') { + steps { + script { + int maxAttempts = 6 + int baseDelay = 10 + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + checkout scm + break + } catch (err) { + if (attempt == maxAttempts) { throw err } + int waitSec = baseDelay * (1 << (attempt - 1)) + echo "Checkout failed (attempt ${attempt}/${maxAttempts}). Waiting ${waitSec}s before retry..." + sleep time: waitSec, unit: 'SECONDS' + } + } + } + } } - } - } - stage('mingw64') { - agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/make_win32.sh' + stage('linux_x86_64') { + steps { + script { retryWithBackoff(2, 5) { sh 'scripts/make_unix.sh' } } + } } - } - } - stage('linux_aarch64') { - agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/make_unix.sh aarch64' + stage('mingw64') { + steps { + script { retryWithBackoff(2, 5) { sh 'scripts/make_win32.sh' } } + } } - } - } - stage('linux_x86_64_test') { - agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/run_tests.sh' + stage('linux_aarch64') { + steps { + script { retryWithBackoff(2, 5) { sh 'scripts/make_unix.sh aarch64' } } + } } - } - } - stage('deliver') { - agent any + stage('linux_x86_64_test') { + steps { + script { retryWithBackoff(2, 5) { sh 'scripts/run_tests.sh' } } + } + } - steps { - sh 'scripts/deliver.sh /mnt/repertory "" "" "" "" 1 1' - sh 'scripts/deliver.sh /mnt/repertory "" aarch64' - sh 'scripts/deliver.sh /mnt/repertory' + stage('deliver') { + steps { + script { + retryWithBackoff(3, 10) { sh 'scripts/deliver.sh /mnt/repertory "" "" "" "" 1 1' } + retryWithBackoff(3, 10) { sh 'scripts/deliver.sh /mnt/repertory "" aarch64' } + retryWithBackoff(3, 10) { sh 'scripts/deliver.sh /mnt/repertory' } + } + } + } } } } } + +def retryWithBackoff(int maxAttempts, int baseDelaySeconds, Closure body) { + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + body() + return + } catch (err) { + if (attempt == maxAttempts) { throw err } + int waitSec = baseDelaySeconds * (1 << (attempt - 1)) + echo "Step failed (attempt ${attempt}/${maxAttempts}). Waiting ${waitSec}s before retry..." + sleep time: waitSec, unit: 'SECONDS' + } + } +} diff --git a/.jenkins_macos b/.jenkins_macos index f226df07..43047334 100644 --- a/.jenkins_macos +++ b/.jenkins_macos @@ -4,57 +4,76 @@ pipeline { agent none environment { - DEVELOPER_PRIVATE_KEY = "${env.HOME}/.ci/repertory/cert/developer.priv" - DEVELOPER_PUBLIC_KEY = "${env.HOME}/.ci/repertory/cert/developer.pub" + DEVELOPER_PRIVATE_KEY = "${env.HOME}/.ci/repertory/cert/developer.priv" + DEVELOPER_PUBLIC_KEY = "${env.HOME}/.ci/repertory/cert/developer.pub" PROJECT_TEST_CONFIG_DIR = "${env.HOME}/.ci/repertory/test" } options { disableConcurrentBuilds() - retry(2) + skipDefaultCheckout() + timestamps() } stages { - stage('darwin_aarch64') { + stage('Build • Test • Deliver') { agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/make_unix.sh aarch64' + stages { + stage('Checkout') { + steps { + script { + int maxAttempts = 6 + int baseDelay = 10 + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + checkout scm + break + } catch (err) { + if (attempt == maxAttempts) { throw err } + int waitSec = baseDelay * (1 << (attempt - 1)) + echo "Checkout failed (attempt ${attempt}/${maxAttempts}). Waiting ${waitSec}s before retry..." + sleep time: waitSec, unit: 'SECONDS' + } + } + } + } } - } - } - stage('darwin_x86_64') { - agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/make_unix.sh x86_64' + stage('darwin_aarch64') { + steps { script { retryWithBackoff(2, 5) { sh 'scripts/make_unix.sh aarch64' } } } } - } - } - stage('darwin_aarch64_test') { - agent any - - steps { - retry(2) { - sleep time: 5, unit: 'SECONDS' - sh 'scripts/run_tests.sh aarch64' + stage('darwin_x86_64') { + steps { script { retryWithBackoff(2, 5) { sh 'scripts/make_unix.sh x86_64' } } } } - } - } - stage('deliver') { - agent any + stage('darwin_aarch64_test') { + steps { script { retryWithBackoff(2, 5) { sh 'scripts/run_tests.sh aarch64' } } } + } - steps { - sh "scripts/deliver.sh ${env.HOME}/mnt/repertory '' aarch64" - sh "scripts/deliver.sh ${env.HOME}/mnt/repertory '' x86_64" + stage('deliver') { + steps { + script { + retryWithBackoff(3, 10) { sh "scripts/deliver.sh ${env.HOME}/mnt/repertory '' aarch64" } + retryWithBackoff(3, 10) { sh "scripts/deliver.sh ${env.HOME}/mnt/repertory '' x86_64" } + } + } + } } } } } + +def retryWithBackoff(int maxAttempts, int baseDelaySeconds, Closure body) { + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + body() + return + } catch (err) { + if (attempt == maxAttempts) { throw err } + int waitSec = baseDelaySeconds * (1 << (attempt - 1)) + echo "Step failed (attempt ${attempt}/${maxAttempts}). Waiting ${waitSec}s before retry..." + sleep time: waitSec, unit: 'SECONDS' + } + } +}