From 79c575a8ef098875ecec98617572d809d36f55c4 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 03:00:05 +0000 Subject: [PATCH 01/43] Add skeleton of Cirrus rbm builds --- .cirrus.yml | 82 +++++++++++++++++++++++++++++++++++ tools/cirrus_build_project.sh | 73 +++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 73 +++++++++++++++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 .cirrus.yml create mode 100755 tools/cirrus_build_project.sh create mode 100755 tools/cirrus_gen_yml.sh diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000..5ae250f --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,82 @@ +nightly_linux_x86_64_download_docker_builder: + timeout_in: 120m + out_nightly_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_nightly_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns nightly linux x86_64 0" + +nightly_linux_x86_64_ncdns_docker_builder: + timeout_in: 120m + out_nightly_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_nightly_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns nightly linux x86_64 1" + depends_on: + - "nightly_linux_x86_64_download" + +nightly_linux_x86_64_ncp11_docker_builder: + timeout_in: 120m + out_nightly_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_nightly_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncp11 nightly linux x86_64 1" + depends_on: + - "nightly_linux_x86_64_ncdns" + +nightly_linux_x86_64_ncprop279_docker_builder: + timeout_in: 120m + out_nightly_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_nightly_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_nightly_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncprop279 nightly linux x86_64 1" + depends_on: + - "nightly_linux_x86_64_ncp11" + diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh new file mode 100755 index 0000000..c96ee17 --- /dev/null +++ b/tools/cirrus_build_project.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euxo pipefail +shopt -s nullglob globstar + +PROJECT=$1 +CHANNEL=$2 +OS=$3 +ARCH=$4 +SHOULD_BUILD=$5 + +echo "Checking VM specs..." +cat /etc/*-release +df -h +lscpu +free -m + +echo "Installing rbm deps..." +apt-get install -y libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc + +echo "Pulling rbm..." +make submodule-update + +echo "Moving caches..." +if [[ -e "./fonts/.git" ]]; then + echo "git_clones/fonts was cached, moving it to the right place..." + mv ./fonts ./git_clones/fonts +else + echo "git_clones/fonts was not cached." + rm -rf ./fonts ./git_clones/fonts +fi +ls ./git_clones + +echo "Checking if project is cached..." +OUTDIR="$(./rbm/rbm showconf $PROJECT output_dir --target $CHANNEL --target ncdns-$OS-$ARCH)" +OUTFILE="$(./rbm/rbm showconf $PROJECT filename --target $CHANNEL --target ncdns-$OS-$ARCH)" +if [[ -e "$OUTDIR/$OUTFILE" ]]; then + echo "Project cache hit, skipping build." + SHOULD_BUILD=0 +else + echo "Project cache miss, proceeding with build." +fi + +# VM has 12 GB of free RAM. Assuming each of the 4 logical cores takes 1 GB +# during build, that leaves us with 8 GB of unutilized RAM. Alas, I'm not sure +# that's enough, so this isn't enabled right now. +#echo "Mounting tmpfs..." +#mount -t tmpfs -o size=8G,nr_inodes=40k,mode=1777 tmpfs ./tmp +#df -h + +if [[ "$SHOULD_BUILD" -eq 1 ]]; then + echo "Building project..." + ./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH" +else + #echo "This is a cache-only task, skipping build." + echo "Skipping build." +fi + +echo "Moving caches..." +if [[ -e "git_clones/fonts" ]]; then + echo "git_clones/fonts is ready to be cached, moving it to the right place..." + mv git_clones/fonts ./ +else + echo "git_clones/fonts is not ready to be not cached." + mkdir -p ./fonts +fi + +# The cache has a size limit, so we need to clean useless data from it. The +# container-images are very large and seem to be fairly harmless to remove. +# Maybe later if we have more pressure to shrink, we could remove the +# debootstrap-images too. +echo "Cleaning cache..." +rm -rfv out/container-image diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh new file mode 100755 index 0000000..9e0709a --- /dev/null +++ b/tools/cirrus_gen_yml.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -euxo pipefail +shopt -s nullglob globstar + +( +#for CHANNEL in release nightly; do +for CHANNEL in nightly; do + OS=linux + ARCH=x86_64 + + # Pre-download tarballs and Git repos + echo "${CHANNEL}_${OS}_${ARCH}_download_docker_builder: + timeout_in: 120m + out_${CHANNEL}_${OS}_${ARCH}_cache: + folder: out + fingerprint_script: + - \"echo out_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p out\" + git_${CHANNEL}_${OS}_${ARCH}_cache: + folder: git_clones + fingerprint_script: + - \"echo git_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p git_clones\" + build_script: + - \"./tools/cirrus_build_project.sh ncdns ${CHANNEL} ${OS} ${ARCH} 0\"" + echo "" + + # TODO fine-tune this list + for PROJECT in ncdns ncp11 ncprop279; do + echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder: + timeout_in: 120m + out_${CHANNEL}_${OS}_${ARCH}_cache: + folder: out + fingerprint_script: + - \"echo out_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p out\" + git_${CHANNEL}_${OS}_${ARCH}_cache: + folder: git_clones + fingerprint_script: + - \"echo git_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p git_clones\" + build_script: + - \"./tools/cirrus_build_project.sh ${PROJECT} ${CHANNEL} ${OS} ${ARCH} 1\"" + + # Depend on previous project + if [[ "$PROJECT" == "ncdns" ]]; then + echo " depends_on: + - \"${CHANNEL}_${OS}_${ARCH}_download\"" + else + echo " depends_on: + - \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT}\"" + fi + + PREV_PROJECT="$PROJECT" + echo "" + done +done +) > .cirrus.yml + +# Timeout issues? +# Might want to increase the timeout -- but we're already using the 2 hour max. +# Might want to bump the CPU count -- but that's blocked by cirrus-ci-docs issue #741. +# Might want to split into smaller project sets. +# What is the CPU count limit? "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 16.0 CPU. docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS). From d62b523d6b181bb00b96c4aea9a97232e4ca9491 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 03:06:57 +0000 Subject: [PATCH 02/43] Cirrus: Namecoin doesn't have a Nightly channel --- .cirrus.yml | 54 ++++++++++++++++++++--------------------- tools/cirrus_gen_yml.sh | 3 +-- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 5ae250f..42bac03 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,82 +1,82 @@ -nightly_linux_x86_64_download_docker_builder: +release_linux_x86_64_download_docker_builder: timeout_in: 120m - out_nightly_linux_x86_64_cache: + out_release_linux_x86_64_cache: folder: out fingerprint_script: - - "echo out_nightly_linux_x86_64" + - "echo out_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p out" - git_nightly_linux_x86_64_cache: + git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: - - "echo git_nightly_linux_x86_64" + - "echo git_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns nightly linux x86_64 0" + - "./tools/cirrus_build_project.sh ncdns release linux x86_64 0" -nightly_linux_x86_64_ncdns_docker_builder: +release_linux_x86_64_ncdns_docker_builder: timeout_in: 120m - out_nightly_linux_x86_64_cache: + out_release_linux_x86_64_cache: folder: out fingerprint_script: - - "echo out_nightly_linux_x86_64" + - "echo out_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p out" - git_nightly_linux_x86_64_cache: + git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: - - "echo git_nightly_linux_x86_64" + - "echo git_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns nightly linux x86_64 1" + - "./tools/cirrus_build_project.sh ncdns release linux x86_64 1" depends_on: - - "nightly_linux_x86_64_download" + - "release_linux_x86_64_download" -nightly_linux_x86_64_ncp11_docker_builder: +release_linux_x86_64_ncp11_docker_builder: timeout_in: 120m - out_nightly_linux_x86_64_cache: + out_release_linux_x86_64_cache: folder: out fingerprint_script: - - "echo out_nightly_linux_x86_64" + - "echo out_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p out" - git_nightly_linux_x86_64_cache: + git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: - - "echo git_nightly_linux_x86_64" + - "echo git_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncp11 nightly linux x86_64 1" + - "./tools/cirrus_build_project.sh ncp11 release linux x86_64 1" depends_on: - - "nightly_linux_x86_64_ncdns" + - "release_linux_x86_64_ncdns" -nightly_linux_x86_64_ncprop279_docker_builder: +release_linux_x86_64_ncprop279_docker_builder: timeout_in: 120m - out_nightly_linux_x86_64_cache: + out_release_linux_x86_64_cache: folder: out fingerprint_script: - - "echo out_nightly_linux_x86_64" + - "echo out_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p out" - git_nightly_linux_x86_64_cache: + git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: - - "echo git_nightly_linux_x86_64" + - "echo git_release_linux_x86_64" reupload_on_changes: true populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncprop279 nightly linux x86_64 1" + - "./tools/cirrus_build_project.sh ncprop279 release linux x86_64 1" depends_on: - - "nightly_linux_x86_64_ncp11" + - "release_linux_x86_64_ncp11" diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 9e0709a..8e1bdc9 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -4,8 +4,7 @@ set -euxo pipefail shopt -s nullglob globstar ( -#for CHANNEL in release nightly; do -for CHANNEL in nightly; do +for CHANNEL in release; do OS=linux ARCH=x86_64 From 430dda3b1a8e83b1099fc8f1349efe6140322dc6 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 03:19:40 +0000 Subject: [PATCH 03/43] Bump tor-browser-build The ancient tor-browser-build was interfering with Cirrus tests. --- tor-browser-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tor-browser-build b/tor-browser-build index 637e37b..41d0330 160000 --- a/tor-browser-build +++ b/tor-browser-build @@ -1 +1 @@ -Subproject commit 637e37bff412b587970791dafdb3174eb359d6b3 +Subproject commit 41d0330f0741d6a78fcea050352f03002ca8520e From 4aaf6bf6c8121fce9e9bf5a304fa50464fea7d94 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 03:51:36 +0000 Subject: [PATCH 04/43] Bump rbm.conf --- rbm.conf | 245 ++++++++++++++++++++++++++++++--- tools/patch-tor-to-namecoin.sh | 6 + 2 files changed, 231 insertions(+), 20 deletions(-) create mode 100755 tools/patch-tor-to-namecoin.sh diff --git a/rbm.conf b/rbm.conf index c4f73e5..e3878b5 100644 --- a/rbm.conf +++ b/rbm.conf @@ -24,7 +24,13 @@ buildconf: git_signtag_opt: '-s' var: + ncdns_version: '10.5a5' + ncdns_build: 'build2' + ncdns_incremental_from: + - 10.5a3 project_name: tor-browser + multi_lingual: 0 + build_mar: 1 # By default, we sort the list of installed packages. This allows sharing # containers with identical list of packages, even if they are not listed # in the same order. In the cases where the installation order is @@ -40,7 +46,8 @@ var: [% END -%] input_files: [% c("input_files_id") %] build: - [% c("build", { filename => 'f', output_dir => '/out', norec => {} }) %] + [% SET step = c("step") -%] + [% c(step, { filename => 'f', output_dir => '/out', norec => {} }) %] container: dir: '[% c("rbm_tmp_dir") %]/rbm-containers/[% sha256(c("build_id")) %]' user: rbm @@ -55,6 +62,76 @@ var: faketime: "faketime -f \"[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]\"" touch: "[% USE date %]touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %]" + locale_ja: ja + locales: + - ar + - ca + - cs + - da + - de + - el + - es-AR + - es-ES + - fa + - fr + - ga-IE + - he + - hu + - id + - is + - it + - '[% c("var/locale_ja") %]' + - ka + - ko + - lt + - mk + - ms + - nb-NO + - nl + - pl + - pt-BR + - ro + - ru + - sv-SE + - th + - tr + - vi + - zh-CN + - zh-TW + locales_mobile: + - ar + - ca + - cs + - da + - de + - el + - es-rAR + - es-rES + - fa + - fr + - ga-rIE + - hu + - in + - is + - it + - iw + - ja + - ka + - ko + - lt + - nb-rNO + - nl + - pl + - pt-rBR + - ro + - ru + - sv-rSE + - th + - tr + - vi + - zh-rCN + - zh-rTW + sign_build: '[% ENV.RBM_SIGN_BUILD %]' sign_build_gpg_opts: '[% ENV.RBM_GPG_OPTS %]' @@ -84,6 +161,11 @@ var: rm -Rf /var/tmp/build /var/tmp/dist [% END -%] + DOCSDIR_project: '[% project %]' + set_PTDIR_DOCSDIR: | + PTDIR="$distdir/TorBrowser/Tor/PluggableTransports" + DOCSDIR="$distdir/TorBrowser/Docs/[% c("var/DOCSDIR_project") %]" + targets: notarget: linux-x86_64 noint: @@ -93,14 +175,50 @@ targets: var: release: 1 channel: release + alpha: + var: + alpha: 1 + channel: alpha + nightly: + fetch: 1 + var: + nightly: 1 + channel: nightly + ncdns_version: | + [% + IF ENV.TORBROWSER_NIGHTLY_VERSION; + GET ENV.TORBROWSER_NIGHTLY_VERSION; + ELSIF c("var/testbuild"); + GET "testbuild"; + ELSE; + GET c("var_p/nightly_ncdns_version"); + END; + -%] + # For nightly builds, we support updates for a limited set of locales + mar_locales: + - de + - es-ES + - fr + - ru + max_ncdns_incremental_from: 2 + build_infos_json: 1 - # The common-stretch target is used to build components that are common to all - # platforms, using Debian stretch. - common-stretch: + ncdns-testbuild: + - testbuild + - alpha + testbuild: + var: + testbuild: 1 + # Don't create mar files to save time + build_mar: 0 + + # The common-buster target is used to build components that are common to all + # platforms, using Debian Buster. + common-buster: var: common: 1 container: - suite: stretch + suite: buster arch: amd64 pre_pkginst: '' deps: @@ -120,6 +238,8 @@ targets: android-armv7: 1 osname: android-armv7 toolchain_arch: arm + abi: armeabi-v7a + cross_prefix: armv7a-linux-androideabi ncdns-android-x86: - android-x86 - android @@ -129,6 +249,8 @@ targets: android-x86: 1 osname: android-x86 toolchain_arch: x86 + abi: x86 + cross_prefix: i686-linux-android ncdns-android-x86_64: - android-x86_64 - android @@ -138,6 +260,8 @@ targets: android-x86_64: 1 osname: android-x86_64 toolchain_arch: x86_64 + abi: x86_64 + cross_prefix: x86_64-linux-android ncdns-android-aarch64: - android-aarch64 - android @@ -147,22 +271,32 @@ targets: android-aarch64: 1 osname: android-aarch64 toolchain_arch: arm64 + abi: arm64-v8a + cross_prefix: aarch64-linux-android android: var: android: 1 compiler: android-toolchain - # API 16 is the minimum we currently support for Tor Browser on Android android_min_api: '[% GET c("var/android_min_api_" _ c("arch")) %]' - # API 21 is the minimum we currently support for arm64 on Android - android_min_api_aarch64: 21 + CC: '[% c("var/cross_prefix") %][% c("var/android_min_api") %]-clang' + CXX: '[% c("var/cross_prefix") %][% c("var/android_min_api") %]-clang' + # API 16 is the minimum we currently support for 32 bit on Android android_min_api_armv7: 16 android_min_api_x86: 16 + # API 21 is the minimum we currently support for 64 bit on Android android_min_api_x86_64: 21 - CC: '$ANDROID_NDK_HOME/[% c("var/toolchain_arch") %]/bin/clang' - CXX: '$ANDROID_NDK_HOME/[% c("var/toolchain_arch") %]/bin/clang++' + android_min_api_aarch64: 21 + # This is needed to get the offline build part for Glean right. + glean_parser: 1.28.6 + # We only build snowflake on the alpha and nightly + # channels for now. + snowflake: '[% c("var/alpha") || c("var/nightly") %]' container: - suite: stretch + suite: buster arch: amd64 + disable_network: + # Disable network in the script for merging GeckoView .aar files + merge_aars: 1 deps: - build-essential - python @@ -170,10 +304,26 @@ targets: - libtool - zip - unzip + - libtinfo5 + configure_opt: '--host=[% c("var/cross_prefix") %] CC=[% c("var/CC") %] [% c("var/configure_opt_project") %]' + pre_pkginst: | + SNAPSHOT_VERSION=20191201T212855Z + OPENJDK_URL=https://snapshot.debian.org/archive/debian/$SNAPSHOT_VERSION/pool/main/o/openjdk-8 + JDK_VERSION=8u232-b09-1~deb9u1_amd64 + apt-get install -y -q wget ca-certificates-java + wget $OPENJDK_URL/openjdk-8-jdk-headless_$JDK_VERSION.deb + wget $OPENJDK_URL/openjdk-8-jre-headless_$JDK_VERSION.deb + echo 92b4f8fb77d793a86e0b03b3b0750592b40a26a5d75956d10dd984a7b3aad4c9 openjdk-8-jdk-headless_$JDK_VERSION.deb | sha256sum -c + echo 84bf52b6cce20ead08b0d5b9fd9b81b4aa3da385ca951b313fe11d5cb1aa4d17 openjdk-8-jre-headless_$JDK_VERSION.deb | sha256sum -c + dpkg -i ./openjdk-8-jre-headless_$JDK_VERSION.deb ./openjdk-8-jdk-headless_$JDK_VERSION.deb ncdns-linux-x86_64: - linux-x86_64 - linux + ncdns-linux-x86_64-asan: + - linux-asan + - linux-x86_64 + - linux ncdns-linux-i686: - linux-i686 - linux @@ -182,19 +332,26 @@ targets: var: linux-x86_64: 1 osname: linux-x86_64 + # We only support RLBox on the nightly channel and x86_64 for now + rlbox: '[% c("var/nightly") %]' linux-i686: arch: i686 var: linux-i686: 1 osname: linux-i686 - configure_opt_i686: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32' - configure_opt: '[% c("var/configure_opt_i686") %]' + configure_opt: '--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 [% c("var/configure_opt_project") %]' linux: var: linux: 1 compiler: gcc + configure_opt: '[% c("var/configure_opt_project") %]' + # We only build snowflake on the alpha and nightly + # channels for now. + snowflake: '[% c("var/alpha") || c("var/nightly") %]' + # Only build Namecoin for linux on nightly + namecoin: '[% c("var/nightly") %]' container: - suite: wheezy + suite: jessie arch: amd64 pre_pkginst: dpkg --add-architecture i386 deps: @@ -208,6 +365,13 @@ targets: - libtool - zip - unzip + linux-asan: + var: + asan: 1 + # RLBox needs clang to create .wasm files but we use mostly GCC for our + # ASan builds. Thus, the compilation currently breaks with RLBox enabled. + # See: tor-browser-build#40063. + rlbox: 0 ncdns-windows-i686: - windows-i686 @@ -219,6 +383,7 @@ targets: arch: x86_64 var: windows-x86_64: 1 + windows-i686: 0 osname: windows-x86_64 # HEASLR is 64 bit only (see bug 12968) flag_HEASLR: '-Wl,--high-entropy-va' @@ -226,19 +391,25 @@ targets: arch: i686 var: windows-i686: 1 + windows-x86_64: 0 osname: windows-i686 + # mingw-w64 does not support SEH on 32bit systems. Be explicit about that. + flag_noSEH: '-Wl,--no-seh' windows: var: windows: 1 container: - suite: stretch + suite: buster arch: amd64 - configure_opt: '--host=[% c("arch") %]-w64-mingw32 CFLAGS="[% c("var/CFLAGS") %]" LDFLAGS="[% c("var/LDFLAGS") %]"' + configure_opt: '--host=[% c("arch") %]-w64-mingw32 CFLAGS="[% c("var/CFLAGS") %]" LDFLAGS="[% c("var/LDFLAGS") %]" [% c("var/configure_opt_project") %]' CFLAGS: '-fstack-protector-strong -fno-strict-overflow -Wno-missing-field-initializers -Wformat -Wformat-security [% c("var/flag_mwindows") %]' - LDFLAGS: '-Wl,--dynamicbase -Wl,--nxcompat -Wl,--enable-reloc-section -Wl,--no-insert-timestamp -lssp -L$gcclibs [% c("var/flag_HEASLR") %] [% c("var/flag_mwindows") %]' + LDFLAGS: '-Wl,--dynamicbase -Wl,--nxcompat -Wl,--enable-reloc-section -Wl,--no-insert-timestamp -lssp -L$gcclibs [% c("var/flag_HEASLR") %] [% c("var/flag_noSEH") %] [% c("var/flag_mwindows") %]' flag_mwindows: '-mwindows' compiler: mingw-w64 faketime_path: /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 + # We only build snowflake on the alpha and nightly + # channels for now. + snowflake: '[% c("var/alpha") || c("var/nightly") %]' deps: - build-essential - python @@ -256,13 +427,19 @@ targets: osx: 1 osname: osx-x86_64 container: - suite: stretch + suite: buster arch: amd64 compiler: 'macosx-toolchain' - configure_opt: '--host=x86_64-apple-darwin11 CC="x86_64-apple-darwin11-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin11-clang++ [% c("var/FLAGS") %]"' - FLAGS: "-target x86_64-apple-darwin11 -B $cctoolsdir -isysroot $sysrootdir" + configure_opt: '--host=x86_64-apple-darwin CC="x86_64-apple-darwin-clang [% c("var/FLAGS") %]" CXX="x86_64-apple-darwin-clang++ [% c("var/FLAGS") %]" [% c("var/configure_opt_project") %]' + FLAGS: "-target x86_64-apple-darwin -B $cctoolsdir -isysroot $sysrootdir" LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie" macosx_deployment_target: '10.9' + locale_ja: ja-JP-mac + # We only support RLBox on the nightly channel for now + rlbox: '[% c("var/nightly") %]' + # We only build snowflake on the alpha and nightly + # channels for now. + snowflake: '[% c("var/alpha") || c("var/nightly") %]' deps: - build-essential - python @@ -271,6 +448,9 @@ targets: - zip - unzip faketime_path: /usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1 + set_PTDIR_DOCSDIR: | + PTDIR="$distdir/Contents/MacOS/Tor/PluggableTransports" + DOCSDIR="$distdir/Contents/Resources/TorBrowser/Docs/[% c("var/DOCSDIR_project") %]" # The no_build_id target can be useful if you want to quickly display # a build template or other option but don't want to spend time to @@ -423,5 +603,30 @@ ENV: my ($out) = capture_exec('sudo', 'runc', '--version'); return $out =~ m/^.*spec: 1\.[0-9]+\.[0-9]+(?:-dev)?$/m; }, + nightly_ncdns_version => sub { + state $version = ''; + return $version if $version; + my (undef, undef, undef, $day, $mon, $year) = gmtime; + $version = sprintf("tbb-nightly.%u.%02u.%02u", $year + 1900, $mon + 1, $day); + return $version; + }, + nightly_ncdns_incremental_from => sub { + my ($project, $options) = @_; + my $nightly_dir = project_config($project, 'basedir', $options) . '/nightly'; + my $current_version = project_config($project, 'var/ncdns_version', $options); + use Path::Tiny; + return [] unless -d $nightly_dir; + my @dirs = sort map { $_->basename } path($nightly_dir)->children(qr/^tbb-nightly\./); + my $nb_incr = project_config($project, ['var', 'max_ncdns_incremental_from'], $options); + my @res; + while ($nb_incr > 0) { + my $dir = pop @dirs; + last unless $dir; + next if $dir eq $current_version; + $nb_incr--; + push @res, $dir; + } + return [@res]; + }, }, ) diff --git a/tools/patch-tor-to-namecoin.sh b/tools/patch-tor-to-namecoin.sh new file mode 100755 index 0000000..7035fe4 --- /dev/null +++ b/tools/patch-tor-to-namecoin.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euxo pipefail +shopt -s nullglob globstar + +cat tor-browser-build/rbm.conf | sed "s/torbrowser/ncdns/g" > rbm.conf From cd60bab86d0f1a5f604c7cbffe98b31956352a98 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 04:05:10 +0000 Subject: [PATCH 05/43] Bump tor-browser-build --- rbm.conf | 8 ++++---- tor-browser-build | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rbm.conf b/rbm.conf index e3878b5..d180f0e 100644 --- a/rbm.conf +++ b/rbm.conf @@ -24,10 +24,10 @@ buildconf: git_signtag_opt: '-s' var: - ncdns_version: '10.5a5' - ncdns_build: 'build2' + ncdns_version: '10.5a10' + ncdns_build: 'build1' ncdns_incremental_from: - - 10.5a3 + - 10.5a8 project_name: tor-browser multi_lingual: 0 build_mar: 1 @@ -287,7 +287,7 @@ targets: android_min_api_x86_64: 21 android_min_api_aarch64: 21 # This is needed to get the offline build part for Glean right. - glean_parser: 1.28.6 + glean_parser: 1.29.0 # We only build snowflake on the alpha and nightly # channels for now. snowflake: '[% c("var/alpha") || c("var/nightly") %]' diff --git a/tor-browser-build b/tor-browser-build index 41d0330..b52fb3f 160000 --- a/tor-browser-build +++ b/tor-browser-build @@ -1 +1 @@ -Subproject commit 41d0330f0741d6a78fcea050352f03002ca8520e +Subproject commit b52fb3fecb60de5efda0f53d127a4eefd31e0508 From d364bf8b098368bee31dca4a1deacd8e1462e5de Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 08:07:41 +0000 Subject: [PATCH 06/43] Bump ncdns version Previous version was incompatible with current Golang version (due to x509-signature-splice patches not applying cleanly). --- projects/gosplicesign/config | 16 ++++++++++++++++ projects/ncdns/config | 10 +++++----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 projects/gosplicesign/config diff --git a/projects/gosplicesign/config b/projects/gosplicesign/config new file mode 100644 index 0000000..aa50258 --- /dev/null +++ b/projects/gosplicesign/config @@ -0,0 +1,16 @@ +version: '[% c("abbrev") %]' +git_url: https://github.com/namecoin/splicesign.git +git_hash: 38bb6fb3ec66c72ecb3a14e1e714768cc6e56ed7 +filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' + +build: '[% c("projects/go/var/build_go_lib") %]' + +var: + container: + use_container: 1 + go_lib: github.com/namecoin/splicesign + +input_files: + - project: container-image + - name: go + project: go diff --git a/projects/ncdns/config b/projects/ncdns/config index 015a927..cd81a7b 100644 --- a/projects/ncdns/config +++ b/projects/ncdns/config @@ -1,6 +1,6 @@ -version: 0.0.10.3 +version: '[% c("abbrev") %]' git_url: https://github.com/namecoin/ncdns.git -git_hash: 'ca0fe5552806a4275f38468c4d3fbcb2cba1cb79' +git_hash: 'c947efb679dd55796e1b2ae405f6ce7d6a5d6b6e' filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' var: @@ -34,8 +34,8 @@ var: enable_namecoin_tlsa: 1 os_go_lib_deps: [] optional_go_lib_deps: + - gosplicesign - gotlsrestrictnss - - gox509signaturesplice optional_go_lib_install: - github.com/namecoin/ncdns/certdehydrate - github.com/namecoin/ncdns/certinject @@ -104,8 +104,8 @@ input_files: - name: gotlsrestrictnss project: gotlsrestrictnss enable: '[% c("var/enable_namecoin_tlsa") %]' - - name: gox509signaturesplice - project: gox509signaturesplice + - name: gosplicesign + project: gosplicesign enable: '[% c("var/enable_namecoin_tlsa") %]' - name: gomadns project: gomadns From 4a4a50d2909b4db3a43d075e95a16ced509e4638 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 08:13:41 +0000 Subject: [PATCH 07/43] Add Cirrus task for goeasyconfig --- .cirrus.yml | 23 ++++++++++++++++++++++- tools/cirrus_gen_yml.sh | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 42bac03..7a27939 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,6 +17,27 @@ release_linux_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux x86_64 0" +release_linux_x86_64_goeasyconfig_docker_builder: + timeout_in: 120m + out_release_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release linux x86_64 1" + depends_on: + - "release_linux_x86_64_download" + release_linux_x86_64_ncdns_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: @@ -36,7 +57,7 @@ release_linux_x86_64_ncdns_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux x86_64 1" depends_on: - - "release_linux_x86_64_download" + - "release_linux_x86_64_goeasyconfig" release_linux_x86_64_ncp11_docker_builder: timeout_in: 120m diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 8e1bdc9..7a3f302 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -30,7 +30,7 @@ for CHANNEL in release; do echo "" # TODO fine-tune this list - for PROJECT in ncdns ncp11 ncprop279; do + for PROJECT in goeasyconfig ncdns ncp11 ncprop279; do echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder: timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: @@ -51,7 +51,7 @@ for CHANNEL in release; do - \"./tools/cirrus_build_project.sh ${PROJECT} ${CHANNEL} ${OS} ${ARCH} 1\"" # Depend on previous project - if [[ "$PROJECT" == "ncdns" ]]; then + if [[ "$PROJECT" == "goeasyconfig" ]]; then echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_download\"" else From 32b4c55a6255ec06ff2850b28b396717add9a7f7 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 18:30:27 +0000 Subject: [PATCH 08/43] Bump tor-browser-build --- tor-browser-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tor-browser-build b/tor-browser-build index b52fb3f..896b7c3 160000 --- a/tor-browser-build +++ b/tor-browser-build @@ -1 +1 @@ -Subproject commit b52fb3fecb60de5efda0f53d127a4eefd31e0508 +Subproject commit 896b7c34db546b55bb466e24e87b01a7eec434e4 From 05cc3820735fd3253ae728a26d3f79c5ea464b1a Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 18:46:13 +0000 Subject: [PATCH 09/43] Cirrus: Add linux-i686 target --- .cirrus.yml | 103 ++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 15 ++++-- 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7a27939..7134e50 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -101,3 +101,106 @@ release_linux_x86_64_ncprop279_docker_builder: depends_on: - "release_linux_x86_64_ncp11" +release_linux_i686_download_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release linux i686 0" + +release_linux_i686_goeasyconfig_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release linux i686 1" + depends_on: + - "release_linux_i686_download" + +release_linux_i686_ncdns_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release linux i686 1" + depends_on: + - "release_linux_i686_goeasyconfig" + +release_linux_i686_ncp11_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncp11 release linux i686 1" + depends_on: + - "release_linux_i686_ncdns" + +release_linux_i686_ncprop279_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncprop279 release linux i686 1" + depends_on: + - "release_linux_i686_ncp11" + diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 7a3f302..d1b8351 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -3,10 +3,9 @@ set -euxo pipefail shopt -s nullglob globstar -( -for CHANNEL in release; do - OS=linux - ARCH=x86_64 +print_os_arch () { + local OS="$1" + local ARCH="$2" # Pre-download tarballs and Git repos echo "${CHANNEL}_${OS}_${ARCH}_download_docker_builder: @@ -59,9 +58,15 @@ for CHANNEL in release; do - \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT}\"" fi - PREV_PROJECT="$PROJECT" + local PREV_PROJECT="$PROJECT" echo "" done +} + +( +for CHANNEL in release; do + print_os_arch linux x86_64 + print_os_arch linux i686 done ) > .cirrus.yml From 3244cf713f7549836b1a7723ca1638ba5fe817f1 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 19:32:09 +0000 Subject: [PATCH 10/43] Cirrus: Add Windows targets --- .cirrus.yml | 206 ++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 2 + 2 files changed, 208 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 7134e50..c1dcb84 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -204,3 +204,209 @@ release_linux_i686_ncprop279_docker_builder: depends_on: - "release_linux_i686_ncp11" +release_windows_x86_64_download_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release windows x86_64 0" + +release_windows_x86_64_goeasyconfig_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release windows x86_64 1" + depends_on: + - "release_windows_x86_64_download" + +release_windows_x86_64_ncdns_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release windows x86_64 1" + depends_on: + - "release_windows_x86_64_goeasyconfig" + +release_windows_x86_64_ncp11_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncp11 release windows x86_64 1" + depends_on: + - "release_windows_x86_64_ncdns" + +release_windows_x86_64_ncprop279_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncprop279 release windows x86_64 1" + depends_on: + - "release_windows_x86_64_ncp11" + +release_windows_i686_download_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release windows i686 0" + +release_windows_i686_goeasyconfig_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release windows i686 1" + depends_on: + - "release_windows_i686_download" + +release_windows_i686_ncdns_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release windows i686 1" + depends_on: + - "release_windows_i686_goeasyconfig" + +release_windows_i686_ncp11_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncp11 release windows i686 1" + depends_on: + - "release_windows_i686_ncdns" + +release_windows_i686_ncprop279_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncprop279 release windows i686 1" + depends_on: + - "release_windows_i686_ncp11" + diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index d1b8351..39f2fc6 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -67,6 +67,8 @@ print_os_arch () { for CHANNEL in release; do print_os_arch linux x86_64 print_os_arch linux i686 + print_os_arch windows x86_64 + print_os_arch windows i686 done ) > .cirrus.yml From 8a14198e83bb0f9d4229a9e3b99df556afd53bc1 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Wed, 17 Feb 2021 23:21:42 +0000 Subject: [PATCH 11/43] Add plain-binaries project --- .cirrus.yml | 84 ++++++++++++++++++++++++++++++++++ Makefile | 30 ++++-------- projects/plain-binaries/build | 15 ++++++ projects/plain-binaries/config | 15 ++++++ tools/cirrus_gen_yml.sh | 2 +- 5 files changed, 125 insertions(+), 21 deletions(-) create mode 100644 projects/plain-binaries/build create mode 100644 projects/plain-binaries/config diff --git a/.cirrus.yml b/.cirrus.yml index c1dcb84..d9f63ec 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -101,6 +101,27 @@ release_linux_x86_64_ncprop279_docker_builder: depends_on: - "release_linux_x86_64_ncp11" +release_linux_x86_64_plain-binaries_docker_builder: + timeout_in: 120m + out_release_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh plain-binaries release linux x86_64 1" + depends_on: + - "release_linux_x86_64_ncprop279" + release_linux_i686_download_docker_builder: timeout_in: 120m out_release_linux_i686_cache: @@ -204,6 +225,27 @@ release_linux_i686_ncprop279_docker_builder: depends_on: - "release_linux_i686_ncp11" +release_linux_i686_plain-binaries_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh plain-binaries release linux i686 1" + depends_on: + - "release_linux_i686_ncprop279" + release_windows_x86_64_download_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: @@ -307,6 +349,27 @@ release_windows_x86_64_ncprop279_docker_builder: depends_on: - "release_windows_x86_64_ncp11" +release_windows_x86_64_plain-binaries_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh plain-binaries release windows x86_64 1" + depends_on: + - "release_windows_x86_64_ncprop279" + release_windows_i686_download_docker_builder: timeout_in: 120m out_release_windows_i686_cache: @@ -410,3 +473,24 @@ release_windows_i686_ncprop279_docker_builder: depends_on: - "release_windows_i686_ncp11" +release_windows_i686_plain-binaries_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh plain-binaries release windows i686 1" + depends_on: + - "release_windows_i686_ncprop279" + diff --git a/Makefile b/Makefile index b240683..9408170 100644 --- a/Makefile +++ b/Makefile @@ -4,59 +4,49 @@ all: release # TODO: Replace the Makefile-based metatarget with an rbm-based one. release: submodule-update release-linux-x86_64 release-linux-i686 release-windows-x86_64 release-windows-i686 release-osx-x86_64 - #$(rbm) build ncdns --target release --target ncdns-all + #$(rbm) build plain-binaries --target release --target ncdns-all release-android-armv7: submodule-update - $(rbm) build ncdns --target release --target ncdns-android-armv7 + $(rbm) build plain-binaries --target release --target ncdns-android-armv7 release-android-x86: submodule-update - $(rbm) build ncdns --target release --target ncdns-android-x86 + $(rbm) build plain-binaries --target release --target ncdns-android-x86 release-android-x86_64: submodule-update - $(rbm) build ncdns --target release --target ncdns-android-x86_64 + $(rbm) build plain-binaries --target release --target ncdns-android-x86_64 release-android-aarch64: submodule-update - $(rbm) build ncdns --target release --target ncdns-android-aarch64 + $(rbm) build plain-binaries --target release --target ncdns-android-aarch64 release-linux-x86_64: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-linux-x86_64 $(rbm) build dnssec-hsts --target release --target ncdns-linux-x86_64 $(rbm) build dnssec-hsts-native --target release --target ncdns-linux-x86_64 - $(rbm) build ncdns --target release --target ncdns-linux-x86_64 - $(rbm) build ncp11 --target release --target ncdns-linux-x86_64 - $(rbm) build ncprop279 --target release --target ncdns-linux-x86_64 + $(rbm) build plain-binaries --target release --target ncdns-linux-x86_64 release-linux-i686: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-linux-i686 $(rbm) build dnssec-hsts --target release --target ncdns-linux-i686 $(rbm) build dnssec-hsts-native --target release --target ncdns-linux-i686 - $(rbm) build ncdns --target release --target ncdns-linux-i686 - $(rbm) build ncp11 --target release --target ncdns-linux-i686 - $(rbm) build ncprop279 --target release --target ncdns-linux-i686 + $(rbm) build plain-binaries --target release --target ncdns-linux-i686 release-windows-i686: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-windows-i686 $(rbm) build dnssec-hsts --target release --target ncdns-windows-i686 $(rbm) build dnssec-hsts-native --target release --target ncdns-windows-i686 - $(rbm) build ncdns --target release --target ncdns-windows-i686 - $(rbm) build ncp11 --target release --target ncdns-windows-i686 - $(rbm) build ncprop279 --target release --target ncdns-windows-i686 + $(rbm) build plain-binaries --target release --target ncdns-windows-i686 release-windows-x86_64: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-windows-x86_64 $(rbm) build dnssec-hsts --target release --target ncdns-windows-x86_64 $(rbm) build dnssec-hsts-native --target release --target ncdns-windows-x86_64 - $(rbm) build ncdns --target release --target ncdns-windows-x86_64 - $(rbm) build ncp11 --target release --target ncdns-windows-x86_64 - $(rbm) build ncprop279 --target release --target ncdns-windows-x86_64 + $(rbm) build plain-binaries --target release --target ncdns-windows-x86_64 release-osx-x86_64: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-osx-x86_64 $(rbm) build dnssec-hsts --target release --target ncdns-osx-x86_64 $(rbm) build dnssec-hsts-native --target release --target ncdns-osx-x86_64 - $(rbm) build ncdns --target release --target ncdns-osx-x86_64 - $(rbm) build ncp11 --target release --target ncdns-osx-x86_64 - $(rbm) build ncprop279 --target release --target ncdns-osx-x86_64 + $(rbm) build plain-binaries --target release --target ncdns-osx-x86_64 submodule-update: ./setup-submodule-symlinks diff --git a/projects/plain-binaries/build b/projects/plain-binaries/build new file mode 100644 index 0000000..649ce69 --- /dev/null +++ b/projects/plain-binaries/build @@ -0,0 +1,15 @@ +#!/bin/bash +[% c("var/set_default_env") -%] + +distdir=/var/tmp/dist/[% project %] +mkdir -p $distdir + +tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncdns') %] +tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncp11') %] +tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncprop279') %] + +cd $distdir +[% c('tar', { + tar_src => [ '.' ], + tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'), + }) %] diff --git a/projects/plain-binaries/config b/projects/plain-binaries/config new file mode 100644 index 0000000..8f54a05 --- /dev/null +++ b/projects/plain-binaries/config @@ -0,0 +1,15 @@ +version: '[% c("var/ncdns_version") %]' +filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz' + +var: + container: + use_container: 1 + +input_files: + - project: container-image + - name: ncdns + project: ncdns + - name: ncp11 + project: ncp11 + - name: ncprop279 + project: ncprop279 diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 39f2fc6..90cf27b 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -29,7 +29,7 @@ print_os_arch () { echo "" # TODO fine-tune this list - for PROJECT in goeasyconfig ncdns ncp11 ncprop279; do + for PROJECT in goeasyconfig ncdns ncp11 ncprop279 plain-binaries; do echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder: timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: From e296c02aad39f718e3973dd3807534934c5fb9ee Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 02:32:18 +0000 Subject: [PATCH 12/43] Add clang-source and ninja These are needed for macOS targets. --- projects/clang-source | 1 + projects/ninja | 1 + 2 files changed, 2 insertions(+) create mode 120000 projects/clang-source create mode 120000 projects/ninja diff --git a/projects/clang-source b/projects/clang-source new file mode 120000 index 0000000..de6f54d --- /dev/null +++ b/projects/clang-source @@ -0,0 +1 @@ +../tor-browser-build/projects/clang-source \ No newline at end of file diff --git a/projects/ninja b/projects/ninja new file mode 120000 index 0000000..3fe4093 --- /dev/null +++ b/projects/ninja @@ -0,0 +1 @@ +../tor-browser-build/projects/ninja \ No newline at end of file From 7c8ab70e44dfa86362ac9f93ba03235da94fcb25 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 02:49:29 +0000 Subject: [PATCH 13/43] Add release project --- .cirrus.yml | 84 ++++++++++++++++++ Makefile | 23 +++-- projects/release/build | 48 +++++++++++ projects/release/config | 184 ++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 2 +- 5 files changed, 328 insertions(+), 13 deletions(-) create mode 100644 projects/release/build create mode 100644 projects/release/config diff --git a/.cirrus.yml b/.cirrus.yml index d9f63ec..182c07a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -122,6 +122,27 @@ release_linux_x86_64_plain-binaries_docker_builder: depends_on: - "release_linux_x86_64_ncprop279" +release_linux_x86_64_release_docker_builder: + timeout_in: 120m + out_release_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh release release linux x86_64 1" + depends_on: + - "release_linux_x86_64_plain-binaries" + release_linux_i686_download_docker_builder: timeout_in: 120m out_release_linux_i686_cache: @@ -246,6 +267,27 @@ release_linux_i686_plain-binaries_docker_builder: depends_on: - "release_linux_i686_ncprop279" +release_linux_i686_release_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh release release linux i686 1" + depends_on: + - "release_linux_i686_plain-binaries" + release_windows_x86_64_download_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: @@ -370,6 +412,27 @@ release_windows_x86_64_plain-binaries_docker_builder: depends_on: - "release_windows_x86_64_ncprop279" +release_windows_x86_64_release_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh release release windows x86_64 1" + depends_on: + - "release_windows_x86_64_plain-binaries" + release_windows_i686_download_docker_builder: timeout_in: 120m out_release_windows_i686_cache: @@ -494,3 +557,24 @@ release_windows_i686_plain-binaries_docker_builder: depends_on: - "release_windows_i686_ncprop279" +release_windows_i686_release_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh release release windows i686 1" + depends_on: + - "release_windows_i686_plain-binaries" + diff --git a/Makefile b/Makefile index 9408170..e59a7b9 100644 --- a/Makefile +++ b/Makefile @@ -2,51 +2,50 @@ rbm=./rbm/rbm all: release -# TODO: Replace the Makefile-based metatarget with an rbm-based one. -release: submodule-update release-linux-x86_64 release-linux-i686 release-windows-x86_64 release-windows-i686 release-osx-x86_64 - #$(rbm) build plain-binaries --target release --target ncdns-all +release: submodule-update + $(rbm) build release --target release --target ncdns-all release-android-armv7: submodule-update - $(rbm) build plain-binaries --target release --target ncdns-android-armv7 + $(rbm) build release --target release --target ncdns-android-armv7 release-android-x86: submodule-update - $(rbm) build plain-binaries --target release --target ncdns-android-x86 + $(rbm) build release --target release --target ncdns-android-x86 release-android-x86_64: submodule-update - $(rbm) build plain-binaries --target release --target ncdns-android-x86_64 + $(rbm) build release --target release --target ncdns-android-x86_64 release-android-aarch64: submodule-update - $(rbm) build plain-binaries --target release --target ncdns-android-aarch64 + $(rbm) build release --target release --target ncdns-android-aarch64 release-linux-x86_64: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-linux-x86_64 $(rbm) build dnssec-hsts --target release --target ncdns-linux-x86_64 $(rbm) build dnssec-hsts-native --target release --target ncdns-linux-x86_64 - $(rbm) build plain-binaries --target release --target ncdns-linux-x86_64 + $(rbm) build release --target release --target ncdns-linux-x86_64 release-linux-i686: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-linux-i686 $(rbm) build dnssec-hsts --target release --target ncdns-linux-i686 $(rbm) build dnssec-hsts-native --target release --target ncdns-linux-i686 - $(rbm) build plain-binaries --target release --target ncdns-linux-i686 + $(rbm) build release --target release --target ncdns-linux-i686 release-windows-i686: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-windows-i686 $(rbm) build dnssec-hsts --target release --target ncdns-windows-i686 $(rbm) build dnssec-hsts-native --target release --target ncdns-windows-i686 - $(rbm) build plain-binaries --target release --target ncdns-windows-i686 + $(rbm) build release --target release --target ncdns-windows-i686 release-windows-x86_64: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-windows-x86_64 $(rbm) build dnssec-hsts --target release --target ncdns-windows-x86_64 $(rbm) build dnssec-hsts-native --target release --target ncdns-windows-x86_64 - $(rbm) build plain-binaries --target release --target ncdns-windows-x86_64 + $(rbm) build release --target release --target ncdns-windows-x86_64 release-osx-x86_64: submodule-update $(rbm) build certdehydrate-dane-rest-api --target release --target ncdns-osx-x86_64 $(rbm) build dnssec-hsts --target release --target ncdns-osx-x86_64 $(rbm) build dnssec-hsts-native --target release --target ncdns-osx-x86_64 - $(rbm) build plain-binaries --target release --target ncdns-osx-x86_64 + $(rbm) build release --target release --target ncdns-osx-x86_64 submodule-update: ./setup-submodule-symlinks diff --git a/projects/release/build b/projects/release/build new file mode 100644 index 0000000..e475230 --- /dev/null +++ b/projects/release/build @@ -0,0 +1,48 @@ +#!/bin/sh +[% c("var/set_default_env") -%] +# reset HOME which was changed by var/set_default_env, for gpg +[% IF ENV.HOME %]export HOME="[% ENV.HOME %]"[% END %] +destdir="[% dest_dir _ '/' _ c("var/publish_dir") %]" +mkdir -p "$destdir" +[% IF c("var/ncdns-android-armv7") -%] + mv [% c('input_files_by_name/android-armv7') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-android-x86") -%] + mv [% c('input_files_by_name/android-x86') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-android-x86_64") -%] + mv [% c('input_files_by_name/android-x86_64') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-android-aarch64") -%] + mv [% c('input_files_by_name/android-aarch64') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-windows-i686") -%] + mv [% c('input_files_by_name/windows-i686') %]/* "$destdir"/ + mv [% c('input_files_by_name/windows-expert-bundle') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-windows-x86_64") -%] + mv [% c('input_files_by_name/windows-x86_64') %]/* "$destdir"/ + mv [% c('input_files_by_name/windows64-expert-bundle') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-osx-x86_64") -%] + mv [% c('input_files_by_name/osx-x86_64') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-linux-i686") -%] + mv [% c('input_files_by_name/linux-i686') %]/* "$destdir"/ +[% END -%] +[% IF c("var/ncdns-linux-x86_64") -%] + mv [% c('input_files_by_name/linux-x86_64') %]/* "$destdir"/ +[% END -%] +cd "$destdir" +cat > .htaccess < sha256sums-unsigned-build.txt +[% IF c("var/sign_build") -%] + gpg -abs [% c("var/sign_build_gpg_opts") %] sha256sums-unsigned-build.txt +[% END -%] +cat sha256sums-unsigned-build.txt diff --git a/projects/release/config b/projects/release/config new file mode 100644 index 0000000..2d542e0 --- /dev/null +++ b/projects/release/config @@ -0,0 +1,184 @@ +# vim: filetype=yaml sw=2 +version: '[% c("var/ncdns_version") %]' +output_dir: release + +var: + signed_status: unsigned + publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/ncdns_build") %]' + containers_target: with_containers + +targets: + ncdns-all: + - ncdns-linux-x86_64 + - ncdns-linux-i686 + - ncdns-windows-i686 + - ncdns-windows-x86_64 + - ncdns-osx-x86_64 + #- ncdns-android-armv7 + #- ncdns-android-x86 + #- ncdns-android-x86_64 + #- ncdns-android-aarch64 + - ncdns-src + ncdns-all-desktop: + - ncdns-linux-x86_64 + - ncdns-linux-i686 + - ncdns-windows-i686 + - ncdns-windows-x86_64 + - ncdns-osx-x86_64 + - ncdns-src + ncdns-all-android: + - ncdns-android-armv7 + - ncdns-android-x86 + - ncdns-android-x86_64 + - ncdns-android-aarch64 + ncdns-android-armv7: + var: + ncdns-android-armv7: 1 + ncdns-android-x86: + var: + ncdns-android-x86: 1 + ncdns-android-x86_64: + var: + ncdns-android-x86_64: 1 + ncdns-android-aarch64: + var: + ncdns-android-aarch64: 1 + ncdns-linux-x86_64: + var: + ncdns-linux-x86_64: 1 + ncdns-linux-x86_64-asan: + var: + ncdns-linux-x86_64: 1 + asan-build: '-asan' + ncdns-linux-i686: + var: + ncdns-linux-i686: 1 + ncdns-windows-i686: + var: + ncdns-windows-i686: 1 + ncdns-windows-x86_64: + var: + ncdns-windows-x86_64: 1 + ncdns-osx-x86_64: + var: + ncdns-osx-x86_64: 1 + ncdns-src: + var: + ncdns-src: '[% ! c("var/testbuild") %]' + ncdns-src-testbuild: + var: + ncdns-src: 1 + + notarget: + - release + - ncdns-all + #- ncdns-all-android + - ncdns-all-desktop + + noversiondir: + var: + publish_dir: '' + + release: + var: + build_target: release + + nightly: + output_dir: 'nightly' + var: + build_target: nightly + publish_dir: '[% c("version") %]' + + alpha: + output_dir: alpha + var: + build_target: alpha + + testbuild: + output_dir: 'testbuild' + var: + testbuild: 1 + build_target: ncdns-testbuild + publish_dir: '' + + no_containers: + var: + containers_target: no_containers + + signed: + var: + signed_status: signed + + create_unsigned_incrementals: + var: + create_unsigned_incrementals: 1 + +input_files: + +# Release + - name: android-armv7 + project: plain-binaries + enable: '[% c("var/ncdns-android-armv7") %]' + target: + - '[% c("var/containers_target") %]' + - '[% c("var/build_target") %]' + - ncdns-android-armv7 + + - name: android-x86 + project: plain-binaries + enable: '[% c("var/ncdns-android-x86") %]' + target: + - '[% c("var/containers_target") %]' + - '[% c("var/build_target") %]' + - ncdns-android-x86 + + - name: android-x86_64 + project: plain-binaries + enable: '[% c("var/ncdns-android-x86_64") %]' + target: + - '[% c("var/containers_target") %]' + - '[% c("var/build_target") %]' + - ncdns-android-x86_64 + + - name: android-aarch64 + project: plain-binaries + enable: '[% c("var/ncdns-android-aarch64") %]' + target: + - '[% c("var/containers_target") %]' + - '[% c("var/build_target") %]' + - ncdns-android-aarch64 + + - name: linux-x86_64 + project: plain-binaries + enable: '[% c("var/ncdns-linux-x86_64") %]' + target: + - '[% c("var/build_target") %]' + - 'ncdns-linux-x86_64[% c("var/asan-build") %]' + + - name: linux-i686 + project: plain-binaries + enable: '[% c("var/ncdns-linux-i686") %]' + target: + - '[% c("var/build_target") %]' + - ncdns-linux-i686 + + - name: windows-i686 + project: plain-binaries + enable: '[% c("var/ncdns-windows-i686") %]' + target: + - '[% c("var/build_target") %]' + - ncdns-windows-i686 + + - name: windows-x86_64 + project: plain-binaries + enable: '[% c("var/ncdns-windows-x86_64") %]' + target: + - '[% c("var/build_target") %]' + - ncdns-windows-x86_64 + + - name: osx-x86_64 + project: plain-binaries + enable: '[% c("var/ncdns-osx-x86_64") %]' + target: + - '[% c("var/build_target") %]' + - ncdns-osx-x86_64 diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 90cf27b..7d53b13 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -29,7 +29,7 @@ print_os_arch () { echo "" # TODO fine-tune this list - for PROJECT in goeasyconfig ncdns ncp11 ncprop279 plain-binaries; do + for PROJECT in goeasyconfig ncdns ncp11 ncprop279 plain-binaries release; do echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder: timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: From 4140483928dea9ed0d30e39cfbf3f37391f0706d Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 02:59:33 +0000 Subject: [PATCH 14/43] Add libtapi project Required for macOS targets. --- projects/libtapi | 1 + 1 file changed, 1 insertion(+) create mode 120000 projects/libtapi diff --git a/projects/libtapi b/projects/libtapi new file mode 120000 index 0000000..bd9e304 --- /dev/null +++ b/projects/libtapi @@ -0,0 +1 @@ +../tor-browser-build/projects/libtapi \ No newline at end of file From eab6460b268cb5fd09fdda1f017cedd120219871 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 03:17:30 +0000 Subject: [PATCH 15/43] Cirrus: Try apt-get twice We were seeing some intermitttent failures here. --- tools/cirrus_build_project.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index c96ee17..c417e18 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -16,7 +16,8 @@ lscpu free -m echo "Installing rbm deps..." -apt-get install -y libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc +APT_DEPS="libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc" +apt-get install -y $APT_DEPS || (sleep 15s && apt-get install -y $APT_DEPS) echo "Pulling rbm..." make submodule-update From 4243f1657be02fb452587ad4c6682a129d2e4088 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 03:18:10 +0000 Subject: [PATCH 16/43] Cirrus: Don't try to cache release project --- tools/cirrus_build_project.sh | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index c417e18..13d7089 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -32,14 +32,18 @@ else fi ls ./git_clones -echo "Checking if project is cached..." -OUTDIR="$(./rbm/rbm showconf $PROJECT output_dir --target $CHANNEL --target ncdns-$OS-$ARCH)" -OUTFILE="$(./rbm/rbm showconf $PROJECT filename --target $CHANNEL --target ncdns-$OS-$ARCH)" -if [[ -e "$OUTDIR/$OUTFILE" ]]; then - echo "Project cache hit, skipping build." - SHOULD_BUILD=0 +if [[ "$PROJECT" == "release" ]]; then + echo "release project is never cached." else - echo "Project cache miss, proceeding with build." + echo "Checking if project is cached..." + OUTDIR="$(./rbm/rbm showconf $PROJECT output_dir --target $CHANNEL --target ncdns-$OS-$ARCH)" + OUTFILE="$(./rbm/rbm showconf $PROJECT filename --target $CHANNEL --target ncdns-$OS-$ARCH)" + if [[ -e "$OUTDIR/$OUTFILE" ]]; then + echo "Project cache hit, skipping build." + SHOULD_BUILD=0 + else + echo "Project cache miss, proceeding with build." + fi fi # VM has 12 GB of free RAM. Assuming each of the 4 logical cores takes 1 GB From 1c0246042791af2d5b8d1a664e9a5f02e3d95e90 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 18:41:13 +0000 Subject: [PATCH 17/43] Add rbm.local.conf.example --- rbm.local.conf.example | 1 + 1 file changed, 1 insertion(+) create mode 120000 rbm.local.conf.example diff --git a/rbm.local.conf.example b/rbm.local.conf.example new file mode 120000 index 0000000..c4cad5d --- /dev/null +++ b/rbm.local.conf.example @@ -0,0 +1 @@ +tor-browser-build/rbm.local.conf.example \ No newline at end of file From 9175e0d2a6b8e3926772387ea67016fe12a78532 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 18:49:46 +0000 Subject: [PATCH 18/43] Cirrus: Print logs from rbm containers --- tools/cirrus_build_project.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index 13d7089..842e2d8 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -22,6 +22,10 @@ apt-get install -y $APT_DEPS || (sleep 15s && apt-get install -y $APT_DEPS) echo "Pulling rbm..." make submodule-update +echo "Configuring rbm..." +# Print logs to Cirrus. +cat rbm.local.conf.example | sed "s/#build_log: '-'/build_log: '-'/g" > rbm.local.conf + echo "Moving caches..." if [[ -e "./fonts/.git" ]]; then echo "git_clones/fonts was cached, moving it to the right place..." From 0113996e01e508fc8831f0963e51201239ca5802 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 19:10:10 +0000 Subject: [PATCH 19/43] Cirrus: Add download task for macOS --- .cirrus.yml | 19 +++++++++++++++++++ tools/cirrus_gen_yml.sh | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 182c07a..1737e88 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -578,3 +578,22 @@ release_windows_i686_release_docker_builder: depends_on: - "release_windows_i686_plain-binaries" +release_osx_x86_64_download_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + build_script: + - "./tools/cirrus_build_project.sh ncdns release osx x86_64 0" + diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 7d53b13..877daf2 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -28,6 +28,11 @@ print_os_arch () { - \"./tools/cirrus_build_project.sh ncdns ${CHANNEL} ${OS} ${ARCH} 0\"" echo "" + # osx from clang onward doesn't work on Cirrus yet + if [[ "$OS" == "osx" ]]; then + return 0 + fi + # TODO fine-tune this list for PROJECT in goeasyconfig ncdns ncp11 ncprop279 plain-binaries release; do echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder: @@ -69,6 +74,7 @@ for CHANNEL in release; do print_os_arch linux i686 print_os_arch windows x86_64 print_os_arch windows i686 + print_os_arch osx x86_64 done ) > .cirrus.yml From ad526b58cb9ea2ade34d3ef4d5a2b001a57aa104 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 19:23:36 +0000 Subject: [PATCH 20/43] Fix build script for release project --- projects/release/build | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/projects/release/build b/projects/release/build index e475230..79a3fbb 100644 --- a/projects/release/build +++ b/projects/release/build @@ -5,33 +5,31 @@ destdir="[% dest_dir _ '/' _ c("var/publish_dir") %]" mkdir -p "$destdir" [% IF c("var/ncdns-android-armv7") -%] - mv [% c('input_files_by_name/android-armv7') %]/* "$destdir"/ + mv [% c('input_files_by_name/android-armv7') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-android-x86") -%] - mv [% c('input_files_by_name/android-x86') %]/* "$destdir"/ + mv [% c('input_files_by_name/android-x86') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-android-x86_64") -%] - mv [% c('input_files_by_name/android-x86_64') %]/* "$destdir"/ + mv [% c('input_files_by_name/android-x86_64') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-android-aarch64") -%] - mv [% c('input_files_by_name/android-aarch64') %]/* "$destdir"/ + mv [% c('input_files_by_name/android-aarch64') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-windows-i686") -%] - mv [% c('input_files_by_name/windows-i686') %]/* "$destdir"/ - mv [% c('input_files_by_name/windows-expert-bundle') %]/* "$destdir"/ + mv [% c('input_files_by_name/windows-i686') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-windows-x86_64") -%] - mv [% c('input_files_by_name/windows-x86_64') %]/* "$destdir"/ - mv [% c('input_files_by_name/windows64-expert-bundle') %]/* "$destdir"/ + mv [% c('input_files_by_name/windows-x86_64') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-osx-x86_64") -%] - mv [% c('input_files_by_name/osx-x86_64') %]/* "$destdir"/ + mv [% c('input_files_by_name/osx-x86_64') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-linux-i686") -%] - mv [% c('input_files_by_name/linux-i686') %]/* "$destdir"/ + mv [% c('input_files_by_name/linux-i686') %] "$destdir"/ [% END -%] [% IF c("var/ncdns-linux-x86_64") -%] - mv [% c('input_files_by_name/linux-x86_64') %]/* "$destdir"/ + mv [% c('input_files_by_name/linux-x86_64') %] "$destdir"/ [% END -%] cd "$destdir" cat > .htaccess < Date: Thu, 18 Feb 2021 20:29:50 +0000 Subject: [PATCH 21/43] Add container-pids tool --- tools/container-pids.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 tools/container-pids.sh diff --git a/tools/container-pids.sh b/tools/container-pids.sh new file mode 100755 index 0000000..3b99595 --- /dev/null +++ b/tools/container-pids.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# https://unix.stackexchange.com/a/299198 +descendent_pids() { + pids=$(pgrep -P $1) + echo $pids + for pid in $pids; do + descendent_pids $pid + done +} + +build_pid=$(pgrep -f '\./build') +echo $build_pid +descendent_pids $build_pid From a72f41b5a3f6703440255e42663706874f7325c0 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 21:18:59 +0000 Subject: [PATCH 22/43] Add container-interrupt script --- tools/container-interrupt.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 tools/container-interrupt.sh diff --git a/tools/container-interrupt.sh b/tools/container-interrupt.sh new file mode 100755 index 0000000..a564ed0 --- /dev/null +++ b/tools/container-interrupt.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +# Make sure tor-browser-build knows why the build stopped. +touch tmp/interrupted + +# Send SIGINT to all processes inside the container. +kill -s SIGINT $(./tools/container-pids.sh) From 0c1c248443a95634ee15a969b7845e2d4b5c5f02 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Thu, 18 Feb 2021 21:58:33 +0000 Subject: [PATCH 23/43] rbm.conf: Handle SIGINT case --- rbm.conf | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/rbm.conf b/rbm.conf index d180f0e..4ea1ab0 100644 --- a/rbm.conf +++ b/rbm.conf @@ -504,6 +504,18 @@ runc: remote_start: | #!/bin/sh set -e + + # Handle SIGINT case + if [ -d '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% c("var/build_id") %]' ] + then + # We previously did this build but it was manually interrupted. + # Restore the container's saved state instead of making a new container. + mkdir -p '[% c("var/container/dir") %]' + rmdir '[% c("var/container/dir") %]' + mv '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% c("var/build_id") %]' '[% c("var/container/dir") %]' + exit + fi + if [ $(ls -1 '[% c("remote_srcdir", { error_if_undef => 1 }) %]/container-image_'* | wc -l) -ne 1 ] then echo "Can't find container image in input files" >&2 @@ -580,6 +592,19 @@ runc: remote_finish: | #!/bin/sh set -e + + # Handle SIGINT case + if [ -e '[% c("rbm_tmp_dir") %]'/../interrupted ] + then + # This build was manually interrupted via tools/container-interrupt.sh. + # Save the container's state instead of deleting, so we can resume + # the build later. + mkdir -p '[% c("rbm_tmp_dir") %]'/../interrupted_dirs + mv '[% c("var/container/dir") %]' '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% c("var/build_id") %]' + rm '[% c("rbm_tmp_dir") %]'/../interrupted + exit + fi + sudo rm -Rf '[% c("var/container/dir", { error_if_undef => 1 }) %]'/rootfs '[% c("var/container/dir", { error_if_undef => 1 }) %]'/config.json rmdir '[% c("var/container/dir") %]' From ea1b2ed18d5fd8bb8bf2cb7a1b963f7b2616ef0a Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:26:51 +0000 Subject: [PATCH 24/43] Add tor-browser-build checkpoint patch --- tools/checkpoints.patch | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tools/checkpoints.patch diff --git a/tools/checkpoints.patch b/tools/checkpoints.patch new file mode 100644 index 0000000..daedec9 --- /dev/null +++ b/tools/checkpoints.patch @@ -0,0 +1,43 @@ +From 25b581c833679939408fa51ce1d43b55ae481d51 Mon Sep 17 00:00:00 2001 +From: Jeremy Rand +Date: Fri, 19 Feb 2021 00:23:40 +0000 +Subject: [PATCH] clang: Support rbm checkpointing + +--- + projects/clang/build | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/projects/clang/build b/projects/clang/build +index c4c16d9..bb8283b 100644 +--- a/projects/clang/build ++++ b/projects/clang/build +@@ -2,6 +2,13 @@ + [% c("var/set_default_env") -%] + distdir=/var/tmp/dist/[% project %] + mkdir -p /var/tmp/dist ++ ++if [[ -e /var/tmp/dist/checkpoint1 ]] ; then ++ set +e ++ source /var/tmp/dist/checkpoint1 ++ set -e ++else ++ + tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %] + export PATH="/var/tmp/dist/cmake/bin:$PATH" + [% IF c("var/linux") || c("var/android") %] +@@ -35,6 +42,12 @@ cd build + [% IF c("var/rlbox") -%]-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \[% END -%] + -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;lld" + [% END -%] ++ ++set > /var/tmp/dist/checkpoint1 ++export -p >> /var/tmp/dist/checkpoint1 ++fi ++ ++cd /var/tmp/build/clang-source/build + make -j[% c("buildconf/num_procs") %] + make install + cd /var/tmp/dist +-- +2.20.1 + From 0796ce65f2fde44cf4c3125c26a6636b9e18fba2 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:30:23 +0000 Subject: [PATCH 25/43] Cirrus: Apply checkpoint patch to tor-browser-build --- tools/cirrus_build_project.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index 842e2d8..fbf5711 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -26,6 +26,11 @@ echo "Configuring rbm..." # Print logs to Cirrus. cat rbm.local.conf.example | sed "s/#build_log: '-'/build_log: '-'/g" > rbm.local.conf +echo "Patching rbm..." +pushd tor-browser-build +patch -p1 < ../tools/checkpoints.patch +popd + echo "Moving caches..." if [[ -e "./fonts/.git" ]]; then echo "git_clones/fonts was cached, moving it to the right place..." From 1e533e5ba6b7805358805bce4109b04a124c7cbf Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:35:53 +0000 Subject: [PATCH 26/43] Add echos to interrupt script --- tools/container-interrupt.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/container-interrupt.sh b/tools/container-interrupt.sh index a564ed0..c8f4ab8 100755 --- a/tools/container-interrupt.sh +++ b/tools/container-interrupt.sh @@ -1,7 +1,11 @@ #!/usr/bin/env bash +echo "Interrupting build container..." + # Make sure tor-browser-build knows why the build stopped. touch tmp/interrupted # Send SIGINT to all processes inside the container. kill -s SIGINT $(./tools/container-pids.sh) + +echo "Interrupted!" From 07b6abb6129e9996e04a2177f487cca3aa700f2a Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:38:54 +0000 Subject: [PATCH 27/43] Cirrus: Add checkpoint background script --- .cirrus.yml | 72 +++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 3 ++ 2 files changed, 75 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 1737e88..e9bf77f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,6 +33,9 @@ release_linux_x86_64_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh goeasyconfig release linux x86_64 1" depends_on: @@ -54,6 +57,9 @@ release_linux_x86_64_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncdns release linux x86_64 1" depends_on: @@ -75,6 +81,9 @@ release_linux_x86_64_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncp11 release linux x86_64 1" depends_on: @@ -96,6 +105,9 @@ release_linux_x86_64_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncprop279 release linux x86_64 1" depends_on: @@ -117,6 +129,9 @@ release_linux_x86_64_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux x86_64 1" depends_on: @@ -138,6 +153,9 @@ release_linux_x86_64_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh release release linux x86_64 1" depends_on: @@ -178,6 +196,9 @@ release_linux_i686_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh goeasyconfig release linux i686 1" depends_on: @@ -199,6 +220,9 @@ release_linux_i686_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncdns release linux i686 1" depends_on: @@ -220,6 +244,9 @@ release_linux_i686_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncp11 release linux i686 1" depends_on: @@ -241,6 +268,9 @@ release_linux_i686_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncprop279 release linux i686 1" depends_on: @@ -262,6 +292,9 @@ release_linux_i686_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux i686 1" depends_on: @@ -283,6 +316,9 @@ release_linux_i686_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh release release linux i686 1" depends_on: @@ -323,6 +359,9 @@ release_windows_x86_64_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh goeasyconfig release windows x86_64 1" depends_on: @@ -344,6 +383,9 @@ release_windows_x86_64_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncdns release windows x86_64 1" depends_on: @@ -365,6 +407,9 @@ release_windows_x86_64_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncp11 release windows x86_64 1" depends_on: @@ -386,6 +431,9 @@ release_windows_x86_64_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncprop279 release windows x86_64 1" depends_on: @@ -407,6 +455,9 @@ release_windows_x86_64_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows x86_64 1" depends_on: @@ -428,6 +479,9 @@ release_windows_x86_64_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh release release windows x86_64 1" depends_on: @@ -468,6 +522,9 @@ release_windows_i686_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh goeasyconfig release windows i686 1" depends_on: @@ -489,6 +546,9 @@ release_windows_i686_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncdns release windows i686 1" depends_on: @@ -510,6 +570,9 @@ release_windows_i686_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncp11 release windows i686 1" depends_on: @@ -531,6 +594,9 @@ release_windows_i686_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh ncprop279 release windows i686 1" depends_on: @@ -552,6 +618,9 @@ release_windows_i686_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows i686 1" depends_on: @@ -573,6 +642,9 @@ release_windows_i686_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - "./tools/cirrus_build_project.sh release release windows i686 1" depends_on: diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 877daf2..b3ee67c 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -51,6 +51,9 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p git_clones\" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh build_script: - \"./tools/cirrus_build_project.sh ${PROJECT} ${CHANNEL} ${OS} ${ARCH} 1\"" From 2c782203341ac085ec882afdb3f4af554788ad70 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:43:32 +0000 Subject: [PATCH 28/43] Cirrus: Don't fail a task if it saved a checkpoint --- tools/cirrus_build_project.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index fbf5711..a8a33dc 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -64,7 +64,8 @@ fi if [[ "$SHOULD_BUILD" -eq 1 ]]; then echo "Building project..." - ./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH" + # If rbm fails, we consider it a success as long as it saved a checkpoint. + ./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH" || ls ./tmp/interrupted_dirs/* else #echo "This is a cache-only task, skipping build." echo "Skipping build." From e0b48020e417cd131cea53fdfdcaf09c6c6aaea8 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:47:47 +0000 Subject: [PATCH 29/43] Cirrus: cache checkpoints --- .cirrus.yml | 168 ++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 7 ++ 2 files changed, 175 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index e9bf77f..99077b4 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,6 +33,13 @@ release_linux_x86_64_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -57,6 +64,13 @@ release_linux_x86_64_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -81,6 +95,13 @@ release_linux_x86_64_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -105,6 +126,13 @@ release_linux_x86_64_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -129,6 +157,13 @@ release_linux_x86_64_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -153,6 +188,13 @@ release_linux_x86_64_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -196,6 +238,13 @@ release_linux_i686_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -220,6 +269,13 @@ release_linux_i686_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -244,6 +300,13 @@ release_linux_i686_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -268,6 +331,13 @@ release_linux_i686_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -292,6 +362,13 @@ release_linux_i686_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -316,6 +393,13 @@ release_linux_i686_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -359,6 +443,13 @@ release_windows_x86_64_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -383,6 +474,13 @@ release_windows_x86_64_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -407,6 +505,13 @@ release_windows_x86_64_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -431,6 +536,13 @@ release_windows_x86_64_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -455,6 +567,13 @@ release_windows_x86_64_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -479,6 +598,13 @@ release_windows_x86_64_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -522,6 +648,13 @@ release_windows_i686_goeasyconfig_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -546,6 +679,13 @@ release_windows_i686_ncdns_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -570,6 +710,13 @@ release_windows_i686_ncp11_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -594,6 +741,13 @@ release_windows_i686_ncprop279_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -618,6 +772,13 @@ release_windows_i686_plain-binaries_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -642,6 +803,13 @@ release_windows_i686_release_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index b3ee67c..afabdff 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -51,6 +51,13 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p git_clones\" + interrupted_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - \"echo interrupted_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p tmp/interrupted_dirs\" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh From 3f8912883ef5054a959c5e4331b0245fe2323ec6 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 01:00:32 +0000 Subject: [PATCH 30/43] Cirrus: Add 2nd iteration for goeasyconfig --- .cirrus.yml | 212 +++++++++++++++++++++++++++++++--------- tools/cirrus_gen_yml.sh | 15 +-- 2 files changed, 177 insertions(+), 50 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 99077b4..8560d28 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -17,7 +17,7 @@ release_linux_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux x86_64 0" -release_linux_x86_64_goeasyconfig_docker_builder: +release_linux_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -48,7 +48,38 @@ release_linux_x86_64_goeasyconfig_docker_builder: depends_on: - "release_linux_x86_64_download" -release_linux_x86_64_ncdns_docker_builder: +release_linux_x86_64_goeasyconfig_2_docker_builder: + timeout_in: 120m + out_release_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release linux x86_64 1" + depends_on: + - "release_linux_x86_64_goeasyconfig_1" + +release_linux_x86_64_ncdns_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -77,9 +108,9 @@ release_linux_x86_64_ncdns_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux x86_64 1" depends_on: - - "release_linux_x86_64_goeasyconfig" + - "release_linux_x86_64_goeasyconfig_2" -release_linux_x86_64_ncp11_docker_builder: +release_linux_x86_64_ncp11_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -108,9 +139,9 @@ release_linux_x86_64_ncp11_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncp11 release linux x86_64 1" depends_on: - - "release_linux_x86_64_ncdns" + - "release_linux_x86_64_ncdns_1" -release_linux_x86_64_ncprop279_docker_builder: +release_linux_x86_64_ncprop279_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -139,9 +170,9 @@ release_linux_x86_64_ncprop279_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncprop279 release linux x86_64 1" depends_on: - - "release_linux_x86_64_ncp11" + - "release_linux_x86_64_ncp11_1" -release_linux_x86_64_plain-binaries_docker_builder: +release_linux_x86_64_plain-binaries_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -170,9 +201,9 @@ release_linux_x86_64_plain-binaries_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux x86_64 1" depends_on: - - "release_linux_x86_64_ncprop279" + - "release_linux_x86_64_ncprop279_1" -release_linux_x86_64_release_docker_builder: +release_linux_x86_64_release_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -201,7 +232,7 @@ release_linux_x86_64_release_docker_builder: build_script: - "./tools/cirrus_build_project.sh release release linux x86_64 1" depends_on: - - "release_linux_x86_64_plain-binaries" + - "release_linux_x86_64_plain-binaries_1" release_linux_i686_download_docker_builder: timeout_in: 120m @@ -222,7 +253,7 @@ release_linux_i686_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux i686 0" -release_linux_i686_goeasyconfig_docker_builder: +release_linux_i686_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -253,7 +284,38 @@ release_linux_i686_goeasyconfig_docker_builder: depends_on: - "release_linux_i686_download" -release_linux_i686_ncdns_docker_builder: +release_linux_i686_goeasyconfig_2_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_linux_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release linux i686 1" + depends_on: + - "release_linux_i686_goeasyconfig_1" + +release_linux_i686_ncdns_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -282,9 +344,9 @@ release_linux_i686_ncdns_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux i686 1" depends_on: - - "release_linux_i686_goeasyconfig" + - "release_linux_i686_goeasyconfig_2" -release_linux_i686_ncp11_docker_builder: +release_linux_i686_ncp11_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -313,9 +375,9 @@ release_linux_i686_ncp11_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncp11 release linux i686 1" depends_on: - - "release_linux_i686_ncdns" + - "release_linux_i686_ncdns_1" -release_linux_i686_ncprop279_docker_builder: +release_linux_i686_ncprop279_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -344,9 +406,9 @@ release_linux_i686_ncprop279_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncprop279 release linux i686 1" depends_on: - - "release_linux_i686_ncp11" + - "release_linux_i686_ncp11_1" -release_linux_i686_plain-binaries_docker_builder: +release_linux_i686_plain-binaries_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -375,9 +437,9 @@ release_linux_i686_plain-binaries_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux i686 1" depends_on: - - "release_linux_i686_ncprop279" + - "release_linux_i686_ncprop279_1" -release_linux_i686_release_docker_builder: +release_linux_i686_release_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -406,7 +468,7 @@ release_linux_i686_release_docker_builder: build_script: - "./tools/cirrus_build_project.sh release release linux i686 1" depends_on: - - "release_linux_i686_plain-binaries" + - "release_linux_i686_plain-binaries_1" release_windows_x86_64_download_docker_builder: timeout_in: 120m @@ -427,7 +489,7 @@ release_windows_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release windows x86_64 0" -release_windows_x86_64_goeasyconfig_docker_builder: +release_windows_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -458,7 +520,38 @@ release_windows_x86_64_goeasyconfig_docker_builder: depends_on: - "release_windows_x86_64_download" -release_windows_x86_64_ncdns_docker_builder: +release_windows_x86_64_goeasyconfig_2_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release windows x86_64 1" + depends_on: + - "release_windows_x86_64_goeasyconfig_1" + +release_windows_x86_64_ncdns_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -487,9 +580,9 @@ release_windows_x86_64_ncdns_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release windows x86_64 1" depends_on: - - "release_windows_x86_64_goeasyconfig" + - "release_windows_x86_64_goeasyconfig_2" -release_windows_x86_64_ncp11_docker_builder: +release_windows_x86_64_ncp11_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -518,9 +611,9 @@ release_windows_x86_64_ncp11_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncp11 release windows x86_64 1" depends_on: - - "release_windows_x86_64_ncdns" + - "release_windows_x86_64_ncdns_1" -release_windows_x86_64_ncprop279_docker_builder: +release_windows_x86_64_ncprop279_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -549,9 +642,9 @@ release_windows_x86_64_ncprop279_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncprop279 release windows x86_64 1" depends_on: - - "release_windows_x86_64_ncp11" + - "release_windows_x86_64_ncp11_1" -release_windows_x86_64_plain-binaries_docker_builder: +release_windows_x86_64_plain-binaries_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -580,9 +673,9 @@ release_windows_x86_64_plain-binaries_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows x86_64 1" depends_on: - - "release_windows_x86_64_ncprop279" + - "release_windows_x86_64_ncprop279_1" -release_windows_x86_64_release_docker_builder: +release_windows_x86_64_release_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -611,7 +704,7 @@ release_windows_x86_64_release_docker_builder: build_script: - "./tools/cirrus_build_project.sh release release windows x86_64 1" depends_on: - - "release_windows_x86_64_plain-binaries" + - "release_windows_x86_64_plain-binaries_1" release_windows_i686_download_docker_builder: timeout_in: 120m @@ -632,7 +725,7 @@ release_windows_i686_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release windows i686 0" -release_windows_i686_goeasyconfig_docker_builder: +release_windows_i686_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -663,7 +756,38 @@ release_windows_i686_goeasyconfig_docker_builder: depends_on: - "release_windows_i686_download" -release_windows_i686_ncdns_docker_builder: +release_windows_i686_goeasyconfig_2_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_windows_i686_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release windows i686 1" + depends_on: + - "release_windows_i686_goeasyconfig_1" + +release_windows_i686_ncdns_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -692,9 +816,9 @@ release_windows_i686_ncdns_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release windows i686 1" depends_on: - - "release_windows_i686_goeasyconfig" + - "release_windows_i686_goeasyconfig_2" -release_windows_i686_ncp11_docker_builder: +release_windows_i686_ncp11_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -723,9 +847,9 @@ release_windows_i686_ncp11_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncp11 release windows i686 1" depends_on: - - "release_windows_i686_ncdns" + - "release_windows_i686_ncdns_1" -release_windows_i686_ncprop279_docker_builder: +release_windows_i686_ncprop279_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -754,9 +878,9 @@ release_windows_i686_ncprop279_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncprop279 release windows i686 1" depends_on: - - "release_windows_i686_ncp11" + - "release_windows_i686_ncp11_1" -release_windows_i686_plain-binaries_docker_builder: +release_windows_i686_plain-binaries_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -785,9 +909,9 @@ release_windows_i686_plain-binaries_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows i686 1" depends_on: - - "release_windows_i686_ncprop279" + - "release_windows_i686_ncprop279_1" -release_windows_i686_release_docker_builder: +release_windows_i686_release_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -816,7 +940,7 @@ release_windows_i686_release_docker_builder: build_script: - "./tools/cirrus_build_project.sh release release windows i686 1" depends_on: - - "release_windows_i686_plain-binaries" + - "release_windows_i686_plain-binaries_1" release_osx_x86_64_download_docker_builder: timeout_in: 120m diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index afabdff..8f3dca9 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -34,8 +34,10 @@ print_os_arch () { fi # TODO fine-tune this list - for PROJECT in goeasyconfig ncdns ncp11 ncprop279 plain-binaries release; do - echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder: + for PROJECT in goeasyconfig.1 goeasyconfig.2 ncdns.1 ncp11.1 ncprop279.1 plain-binaries.1 release.1; do + PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) + PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2) + echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_docker_builder: timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -62,18 +64,19 @@ print_os_arch () { - sleep 110m - ./tools/container-interrupt.sh build_script: - - \"./tools/cirrus_build_project.sh ${PROJECT} ${CHANNEL} ${OS} ${ARCH} 1\"" + - \"./tools/cirrus_build_project.sh ${PROJECT_BASE} ${CHANNEL} ${OS} ${ARCH} 1\"" # Depend on previous project - if [[ "$PROJECT" == "goeasyconfig" ]]; then + if [[ "$PROJECT" == "goeasyconfig.1" ]]; then echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_download\"" else echo " depends_on: - - \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT}\"" + - \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT_BASE}_${PREV_PROJECT_ITER}\"" fi - local PREV_PROJECT="$PROJECT" + local PREV_PROJECT_BASE="$PROJECT_BASE" + local PREV_PROJECT_ITER="$PROJECT_ITER" echo "" done } From f0a34d61a42d8bb858428aaba6018b68b22890bd Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 01:02:10 +0000 Subject: [PATCH 31/43] Cirrus: Enable builds for macOS --- .cirrus.yml | 217 ++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 5 - 2 files changed, 217 insertions(+), 5 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 8560d28..8f4c6be 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -961,3 +961,220 @@ release_osx_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release osx x86_64 0" +release_osx_x86_64_goeasyconfig_1_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release osx x86_64 1" + depends_on: + - "release_osx_x86_64_download" + +release_osx_x86_64_goeasyconfig_2_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh goeasyconfig release osx x86_64 1" + depends_on: + - "release_osx_x86_64_goeasyconfig_1" + +release_osx_x86_64_ncdns_1_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh ncdns release osx x86_64 1" + depends_on: + - "release_osx_x86_64_goeasyconfig_2" + +release_osx_x86_64_ncp11_1_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh ncp11 release osx x86_64 1" + depends_on: + - "release_osx_x86_64_ncdns_1" + +release_osx_x86_64_ncprop279_1_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh ncprop279 release osx x86_64 1" + depends_on: + - "release_osx_x86_64_ncp11_1" + +release_osx_x86_64_plain-binaries_1_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh plain-binaries release osx x86_64 1" + depends_on: + - "release_osx_x86_64_ncprop279_1" + +release_osx_x86_64_release_1_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs + fingerprint_script: + - "echo interrupted_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p tmp/interrupted_dirs" + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh release release osx x86_64 1" + depends_on: + - "release_osx_x86_64_plain-binaries_1" + diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 8f3dca9..5cb2dd1 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -28,11 +28,6 @@ print_os_arch () { - \"./tools/cirrus_build_project.sh ncdns ${CHANNEL} ${OS} ${ARCH} 0\"" echo "" - # osx from clang onward doesn't work on Cirrus yet - if [[ "$OS" == "osx" ]]; then - return 0 - fi - # TODO fine-tune this list for PROJECT in goeasyconfig.1 goeasyconfig.2 ncdns.1 ncp11.1 ncprop279.1 plain-binaries.1 release.1; do PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) From 05fb0e2b1dc4c9899db884bf87612b13ad91dcc9 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 01:24:42 +0000 Subject: [PATCH 32/43] Cirrus: fix empty checkpoint directory detection --- tools/cirrus_build_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index a8a33dc..e66c85b 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -65,7 +65,7 @@ fi if [[ "$SHOULD_BUILD" -eq 1 ]]; then echo "Building project..." # If rbm fails, we consider it a success as long as it saved a checkpoint. - ./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH" || ls ./tmp/interrupted_dirs/* + ./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH" || [ ! -z "$(ls -A ./tmp/interrupted_dirs/)" ] else #echo "This is a cache-only task, skipping build." echo "Skipping build." From e18caa0c79d0a7961bf5e9bfaf8e0f615baaea1a Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 00:36:10 +0000 Subject: [PATCH 33/43] Fix interrupted cache dir for projects without a version --- rbm.conf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rbm.conf b/rbm.conf index 4ea1ab0..cd2f037 100644 --- a/rbm.conf +++ b/rbm.conf @@ -506,13 +506,13 @@ runc: set -e # Handle SIGINT case - if [ -d '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% c("var/build_id") %]' ] + if [ -d '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% sha256(dest_dir _ '/' _ c("filename")) %]' ] then # We previously did this build but it was manually interrupted. # Restore the container's saved state instead of making a new container. mkdir -p '[% c("var/container/dir") %]' rmdir '[% c("var/container/dir") %]' - mv '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% c("var/build_id") %]' '[% c("var/container/dir") %]' + mv '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% sha256(dest_dir _ '/' _ c("filename")) %]' '[% c("var/container/dir") %]' exit fi @@ -600,7 +600,7 @@ runc: # Save the container's state instead of deleting, so we can resume # the build later. mkdir -p '[% c("rbm_tmp_dir") %]'/../interrupted_dirs - mv '[% c("var/container/dir") %]' '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% c("var/build_id") %]' + mv '[% c("var/container/dir") %]' '[% c("rbm_tmp_dir") %]'/../interrupted_dirs/'[% sha256(dest_dir _ '/' _ c("filename")) %]' rm '[% c("rbm_tmp_dir") %]'/../interrupted exit fi From ef3d6a1df12331a8d9193f2fa805aaf128fcde39 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 04:30:18 +0000 Subject: [PATCH 34/43] Make checkpoint cache upload/download more robust --- .cirrus.yml | 630 +++++++++++++++++++++-------- tools/cirrus_build_project.sh | 6 + tools/cirrus_gen_yml.sh | 18 +- tools/cirrus_pack_interrupted.sh | 22 + tools/cirrus_unpack_interrupted.sh | 16 + 5 files changed, 512 insertions(+), 180 deletions(-) create mode 100755 tools/cirrus_pack_interrupted.sh create mode 100755 tools/cirrus_unpack_interrupted.sh diff --git a/.cirrus.yml b/.cirrus.yml index 8f4c6be..9b9b901 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,13 +33,21 @@ release_linux_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -64,13 +72,21 @@ release_linux_x86_64_goeasyconfig_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -95,13 +111,21 @@ release_linux_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -126,13 +150,21 @@ release_linux_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -157,13 +189,21 @@ release_linux_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -188,13 +228,21 @@ release_linux_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -219,13 +267,21 @@ release_linux_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_x86_64" + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -269,13 +325,21 @@ release_linux_i686_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -300,13 +364,21 @@ release_linux_i686_goeasyconfig_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -331,13 +403,21 @@ release_linux_i686_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -362,13 +442,21 @@ release_linux_i686_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -393,13 +481,21 @@ release_linux_i686_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -424,13 +520,21 @@ release_linux_i686_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -455,13 +559,21 @@ release_linux_i686_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_linux_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_linux_i686" + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -505,13 +617,21 @@ release_windows_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -536,13 +656,21 @@ release_windows_x86_64_goeasyconfig_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -567,13 +695,21 @@ release_windows_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -598,13 +734,21 @@ release_windows_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -629,13 +773,21 @@ release_windows_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -660,13 +812,21 @@ release_windows_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -691,13 +851,21 @@ release_windows_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_x86_64" + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -741,13 +909,21 @@ release_windows_i686_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -772,13 +948,21 @@ release_windows_i686_goeasyconfig_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -803,13 +987,21 @@ release_windows_i686_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -834,13 +1026,21 @@ release_windows_i686_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -865,13 +1065,21 @@ release_windows_i686_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -896,13 +1104,21 @@ release_windows_i686_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -927,13 +1143,21 @@ release_windows_i686_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_windows_i686_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_windows_i686" + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -977,13 +1201,21 @@ release_osx_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -1008,13 +1240,21 @@ release_osx_x86_64_goeasyconfig_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -1039,13 +1279,21 @@ release_osx_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -1070,13 +1318,21 @@ release_osx_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -1101,13 +1357,21 @@ release_osx_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -1132,13 +1396,21 @@ release_osx_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh @@ -1163,13 +1435,21 @@ release_osx_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" - interrupted_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - "echo interrupted_release_osx_x86_64" + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" reupload_on_changes: true - populate_script: - - "mkdir -p tmp/interrupted_dirs" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index e66c85b..b128b4a 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -41,6 +41,9 @@ else fi ls ./git_clones +echo "Unpacking interrupted cache..." +./tools/cirrus_unpack_interrupted.sh || true + if [[ "$PROJECT" == "release" ]]; then echo "release project is never cached." else @@ -86,3 +89,6 @@ fi # debootstrap-images too. echo "Cleaning cache..." rm -rfv out/container-image + +echo "Packing interrupted cache..." +./tools/cirrus_pack_interrupted.sh || true diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 5cb2dd1..e3a9d54 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -48,13 +48,21 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p git_clones\" - interrupted_${CHANNEL}_${OS}_${ARCH}_cache: - folder: tmp/interrupted_dirs + interrupted_aa_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder fingerprint_script: - - \"echo interrupted_${CHANNEL}_${OS}_${ARCH}\" + - \"echo interrupted_aa_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + interrupted_ab_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - \"echo interrupted_ab_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + interrupted_ac_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - \"echo interrupted_ac_${CHANNEL}_${OS}_${ARCH}\" reupload_on_changes: true - populate_script: - - \"mkdir -p tmp/interrupted_dirs\" checkpoint_background_script: - sleep 110m - ./tools/container-interrupt.sh diff --git a/tools/cirrus_pack_interrupted.sh b/tools/cirrus_pack_interrupted.sh new file mode 100755 index 0000000..1c7682a --- /dev/null +++ b/tools/cirrus_pack_interrupted.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euxo pipefail +shopt -s failglob + +pushd tmp + +mkdir -p interrupted_dirs +tar -caf "interrupted_dirs.tar.gz" interrupted_dirs + +CHUNKS=3 + +split --number=$CHUNKS interrupted_dirs.tar.gz interrupted_dirs.tar.gz.part + +rm interrupted_dirs.tar.gz + +for PART in interrupted_dirs.tar.gz.part* ; do + mkdir -p $PART.folder + mv $PART $PART.folder/ +done + +popd diff --git a/tools/cirrus_unpack_interrupted.sh b/tools/cirrus_unpack_interrupted.sh new file mode 100755 index 0000000..acfd095 --- /dev/null +++ b/tools/cirrus_unpack_interrupted.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euxo pipefail +shopt -s failglob + +pushd tmp + +cat interrupted_dirs.tar.gz.part*.folder/interrupted_dirs.tar.gz.part* > interrupted_dirs.tar.gz + +rm -rf interrupted_dirs.tar.gz.part*.folder + +tar -xaf interrupted_dirs.tar.gz + +rm interrupted_dirs.tar.gz + +popd From e6713ec05012cca0ad2a1f60e38de77aa76e0f5a Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 05:57:38 +0000 Subject: [PATCH 35/43] Cirrus: add ncp11 and ncprop279 to download task --- .cirrus.yml | 10 +++++----- tools/cirrus_gen_yml.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 9b9b901..ffb49f3 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -15,7 +15,7 @@ release_linux_x86_64_download_docker_builder: populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns release linux x86_64 0" + - "./tools/cirrus_build_project.sh plain-binaries release linux x86_64 0" release_linux_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m @@ -307,7 +307,7 @@ release_linux_i686_download_docker_builder: populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns release linux i686 0" + - "./tools/cirrus_build_project.sh plain-binaries release linux i686 0" release_linux_i686_goeasyconfig_1_docker_builder: timeout_in: 120m @@ -599,7 +599,7 @@ release_windows_x86_64_download_docker_builder: populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns release windows x86_64 0" + - "./tools/cirrus_build_project.sh plain-binaries release windows x86_64 0" release_windows_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m @@ -891,7 +891,7 @@ release_windows_i686_download_docker_builder: populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns release windows i686 0" + - "./tools/cirrus_build_project.sh plain-binaries release windows i686 0" release_windows_i686_goeasyconfig_1_docker_builder: timeout_in: 120m @@ -1183,7 +1183,7 @@ release_osx_x86_64_download_docker_builder: populate_script: - "mkdir -p git_clones" build_script: - - "./tools/cirrus_build_project.sh ncdns release osx x86_64 0" + - "./tools/cirrus_build_project.sh plain-binaries release osx x86_64 0" release_osx_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index e3a9d54..4dff529 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -25,7 +25,7 @@ print_os_arch () { populate_script: - \"mkdir -p git_clones\" build_script: - - \"./tools/cirrus_build_project.sh ncdns ${CHANNEL} ${OS} ${ARCH} 0\"" + - \"./tools/cirrus_build_project.sh plain-binaries ${CHANNEL} ${OS} ${ARCH} 0\"" echo "" # TODO fine-tune this list From 2a83d0a538dad8e83d772e70a87eacc7b51a66fe Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 08:53:52 +0000 Subject: [PATCH 36/43] Patch macosx-toolchain for checkpoints --- tools/checkpoints.patch | 45 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/tools/checkpoints.patch b/tools/checkpoints.patch index daedec9..53fc49e 100644 --- a/tools/checkpoints.patch +++ b/tools/checkpoints.patch @@ -1,7 +1,7 @@ From 25b581c833679939408fa51ce1d43b55ae481d51 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Fri, 19 Feb 2021 00:23:40 +0000 -Subject: [PATCH] clang: Support rbm checkpointing +Subject: [PATCH 1/2] clang: Support rbm checkpointing --- projects/clang/build | 13 +++++++++++++ @@ -41,3 +41,46 @@ index c4c16d9..bb8283b 100644 -- 2.20.1 + +From ab0c171fe1c82efed5cbaf657b06d7103f209cb9 Mon Sep 17 00:00:00 2001 +From: Jeremy Rand +Date: Sat, 20 Feb 2021 08:51:15 +0000 +Subject: [PATCH 2/2] macosx-toolchain: Support rbm checkpointing + +--- + projects/macosx-toolchain/build | 13 +++++++++++++ + 1 file changed, 13 insertions(+) + +diff --git a/projects/macosx-toolchain/build b/projects/macosx-toolchain/build +index 5716a9e..94f6df1 100644 +--- a/projects/macosx-toolchain/build ++++ b/projects/macosx-toolchain/build +@@ -1,5 +1,12 @@ + #!/bin/bash + [% c("var/set_default_env") -%] ++ ++if [[ -e /var/tmp/dist/checkpoint1 ]] ; then ++ set +e ++ source /var/tmp/dist/checkpoint1 ++ set -e ++else ++ + builddir=/var/tmp/build + mkdir $builddir + distdir=/var/tmp/dist/[% project %] +@@ -65,6 +72,12 @@ cmake -GNinja \ + -DDARWIN_osx_ARCHS=x86_64 \ + -DDARWIN_osx_SYSROOT=$sysrootdir \ + -DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin $builddir/clang-source ++ ++set > /var/tmp/dist/checkpoint1 ++export -p >> /var/tmp/dist/checkpoint1 ++fi ++ ++cd $builddir/clang-source/build + ninja install -v + + # We now have a native macosx64 toolchain. +-- +2.20.1 + From a5596fde6d1010047d73476a9517823b7f1c2068 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 09:01:15 +0000 Subject: [PATCH 37/43] Cirrus: clear checkpoint on download task And also on any task where the output is already cached. --- tools/cirrus_build_project.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index b128b4a..27acdc4 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -72,6 +72,9 @@ if [[ "$SHOULD_BUILD" -eq 1 ]]; then else #echo "This is a cache-only task, skipping build." echo "Skipping build." + + echo "Clearing interrupted cache..." + rm -rf ./tmp/interrupted_dirs/* || true fi echo "Moving caches..." From bb777f967419a6438aee676035b954a76b567159 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 09:10:20 +0000 Subject: [PATCH 38/43] Cirrus: Expose checkpoint cache to download task This allows the checkpoint clearing to actually take effect. --- .cirrus.yml | 75 +++++++++++++++++++++++++++++++++++++++++ tools/cirrus_gen_yml.sh | 15 +++++++++ 2 files changed, 90 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index ffb49f3..f354753 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -14,6 +14,21 @@ release_linux_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" + reupload_on_changes: true build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux x86_64 0" @@ -306,6 +321,21 @@ release_linux_i686_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" + reupload_on_changes: true build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux i686 0" @@ -598,6 +628,21 @@ release_windows_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" + reupload_on_changes: true build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows x86_64 0" @@ -890,6 +935,21 @@ release_windows_i686_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" + reupload_on_changes: true build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows i686 0" @@ -1182,6 +1242,21 @@ release_osx_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p git_clones" + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" + reupload_on_changes: true build_script: - "./tools/cirrus_build_project.sh plain-binaries release osx x86_64 0" diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 4dff529..a2fa53a 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -24,6 +24,21 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p git_clones\" + interrupted_aa_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - \"echo interrupted_aa_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + interrupted_ab_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - \"echo interrupted_ab_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + interrupted_ac_${CHANNEL}_${OS}_${ARCH}_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - \"echo interrupted_ac_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true build_script: - \"./tools/cirrus_build_project.sh plain-binaries ${CHANNEL} ${OS} ${ARCH} 0\"" echo "" From aa27f987b4eed62a1c763de3b902e404b90b0820 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 18:00:35 +0000 Subject: [PATCH 39/43] Cirrus: Factor out compiler project into its own tasks --- .cirrus.yml | 245 ++++++++++++++++++++++++++++++++++++---- tools/cirrus_gen_yml.sh | 18 ++- 2 files changed, 236 insertions(+), 27 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index f354753..c3f2945 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -32,7 +32,7 @@ release_linux_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux x86_64 0" -release_linux_x86_64_goeasyconfig_1_docker_builder: +release_linux_x86_64_gcc_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -67,11 +67,50 @@ release_linux_x86_64_goeasyconfig_1_docker_builder: - sleep 110m - ./tools/container-interrupt.sh build_script: - - "./tools/cirrus_build_project.sh goeasyconfig release linux x86_64 1" + - "./tools/cirrus_build_project.sh gcc release linux x86_64 1" depends_on: - "release_linux_x86_64_download" -release_linux_x86_64_goeasyconfig_2_docker_builder: +release_linux_x86_64_gcc_2_docker_builder: + timeout_in: 120m + out_release_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" + reupload_on_changes: true + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh gcc release linux x86_64 1" + depends_on: + - "release_linux_x86_64_gcc_1" + +release_linux_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -108,7 +147,7 @@ release_linux_x86_64_goeasyconfig_2_docker_builder: build_script: - "./tools/cirrus_build_project.sh goeasyconfig release linux x86_64 1" depends_on: - - "release_linux_x86_64_goeasyconfig_1" + - "release_linux_x86_64_gcc_2" release_linux_x86_64_ncdns_1_docker_builder: timeout_in: 120m @@ -147,7 +186,7 @@ release_linux_x86_64_ncdns_1_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux x86_64 1" depends_on: - - "release_linux_x86_64_goeasyconfig_2" + - "release_linux_x86_64_goeasyconfig_1" release_linux_x86_64_ncp11_1_docker_builder: timeout_in: 120m @@ -339,7 +378,7 @@ release_linux_i686_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release linux i686 0" -release_linux_i686_goeasyconfig_1_docker_builder: +release_linux_i686_gcc_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -374,11 +413,50 @@ release_linux_i686_goeasyconfig_1_docker_builder: - sleep 110m - ./tools/container-interrupt.sh build_script: - - "./tools/cirrus_build_project.sh goeasyconfig release linux i686 1" + - "./tools/cirrus_build_project.sh gcc release linux i686 1" depends_on: - "release_linux_i686_download" -release_linux_i686_goeasyconfig_2_docker_builder: +release_linux_i686_gcc_2_docker_builder: + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" + reupload_on_changes: true + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh gcc release linux i686 1" + depends_on: + - "release_linux_i686_gcc_1" + +release_linux_i686_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -415,7 +493,7 @@ release_linux_i686_goeasyconfig_2_docker_builder: build_script: - "./tools/cirrus_build_project.sh goeasyconfig release linux i686 1" depends_on: - - "release_linux_i686_goeasyconfig_1" + - "release_linux_i686_gcc_2" release_linux_i686_ncdns_1_docker_builder: timeout_in: 120m @@ -454,7 +532,7 @@ release_linux_i686_ncdns_1_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release linux i686 1" depends_on: - - "release_linux_i686_goeasyconfig_2" + - "release_linux_i686_goeasyconfig_1" release_linux_i686_ncp11_1_docker_builder: timeout_in: 120m @@ -646,7 +724,7 @@ release_windows_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows x86_64 0" -release_windows_x86_64_goeasyconfig_1_docker_builder: +release_windows_x86_64_mingw-w64_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -681,11 +759,50 @@ release_windows_x86_64_goeasyconfig_1_docker_builder: - sleep 110m - ./tools/container-interrupt.sh build_script: - - "./tools/cirrus_build_project.sh goeasyconfig release windows x86_64 1" + - "./tools/cirrus_build_project.sh mingw-w64 release windows x86_64 1" depends_on: - "release_windows_x86_64_download" -release_windows_x86_64_goeasyconfig_2_docker_builder: +release_windows_x86_64_mingw-w64_2_docker_builder: + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" + reupload_on_changes: true + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh mingw-w64 release windows x86_64 1" + depends_on: + - "release_windows_x86_64_mingw-w64_1" + +release_windows_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -722,7 +839,7 @@ release_windows_x86_64_goeasyconfig_2_docker_builder: build_script: - "./tools/cirrus_build_project.sh goeasyconfig release windows x86_64 1" depends_on: - - "release_windows_x86_64_goeasyconfig_1" + - "release_windows_x86_64_mingw-w64_2" release_windows_x86_64_ncdns_1_docker_builder: timeout_in: 120m @@ -761,7 +878,7 @@ release_windows_x86_64_ncdns_1_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release windows x86_64 1" depends_on: - - "release_windows_x86_64_goeasyconfig_2" + - "release_windows_x86_64_goeasyconfig_1" release_windows_x86_64_ncp11_1_docker_builder: timeout_in: 120m @@ -953,7 +1070,7 @@ release_windows_i686_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release windows i686 0" -release_windows_i686_goeasyconfig_1_docker_builder: +release_windows_i686_mingw-w64_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -988,11 +1105,50 @@ release_windows_i686_goeasyconfig_1_docker_builder: - sleep 110m - ./tools/container-interrupt.sh build_script: - - "./tools/cirrus_build_project.sh goeasyconfig release windows i686 1" + - "./tools/cirrus_build_project.sh mingw-w64 release windows i686 1" depends_on: - "release_windows_i686_download" -release_windows_i686_goeasyconfig_2_docker_builder: +release_windows_i686_mingw-w64_2_docker_builder: + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" + reupload_on_changes: true + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh mingw-w64 release windows i686 1" + depends_on: + - "release_windows_i686_mingw-w64_1" + +release_windows_i686_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -1029,7 +1185,7 @@ release_windows_i686_goeasyconfig_2_docker_builder: build_script: - "./tools/cirrus_build_project.sh goeasyconfig release windows i686 1" depends_on: - - "release_windows_i686_goeasyconfig_1" + - "release_windows_i686_mingw-w64_2" release_windows_i686_ncdns_1_docker_builder: timeout_in: 120m @@ -1068,7 +1224,7 @@ release_windows_i686_ncdns_1_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release windows i686 1" depends_on: - - "release_windows_i686_goeasyconfig_2" + - "release_windows_i686_goeasyconfig_1" release_windows_i686_ncp11_1_docker_builder: timeout_in: 120m @@ -1260,7 +1416,7 @@ release_osx_x86_64_download_docker_builder: build_script: - "./tools/cirrus_build_project.sh plain-binaries release osx x86_64 0" -release_osx_x86_64_goeasyconfig_1_docker_builder: +release_osx_x86_64_macosx-toolchain_1_docker_builder: timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -1295,11 +1451,50 @@ release_osx_x86_64_goeasyconfig_1_docker_builder: - sleep 110m - ./tools/container-interrupt.sh build_script: - - "./tools/cirrus_build_project.sh goeasyconfig release osx x86_64 1" + - "./tools/cirrus_build_project.sh macosx-toolchain release osx x86_64 1" depends_on: - "release_osx_x86_64_download" -release_osx_x86_64_goeasyconfig_2_docker_builder: +release_osx_x86_64_macosx-toolchain_2_docker_builder: + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" + reupload_on_changes: true + checkpoint_background_script: + - sleep 110m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh macosx-toolchain release osx x86_64 1" + depends_on: + - "release_osx_x86_64_macosx-toolchain_1" + +release_osx_x86_64_goeasyconfig_1_docker_builder: timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -1336,7 +1531,7 @@ release_osx_x86_64_goeasyconfig_2_docker_builder: build_script: - "./tools/cirrus_build_project.sh goeasyconfig release osx x86_64 1" depends_on: - - "release_osx_x86_64_goeasyconfig_1" + - "release_osx_x86_64_macosx-toolchain_2" release_osx_x86_64_ncdns_1_docker_builder: timeout_in: 120m @@ -1375,7 +1570,7 @@ release_osx_x86_64_ncdns_1_docker_builder: build_script: - "./tools/cirrus_build_project.sh ncdns release osx x86_64 1" depends_on: - - "release_osx_x86_64_goeasyconfig_2" + - "release_osx_x86_64_goeasyconfig_1" release_osx_x86_64_ncp11_1_docker_builder: timeout_in: 120m diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index a2fa53a..56fbb15 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -44,8 +44,22 @@ print_os_arch () { echo "" # TODO fine-tune this list - for PROJECT in goeasyconfig.1 goeasyconfig.2 ncdns.1 ncp11.1 ncprop279.1 plain-binaries.1 release.1; do + for PROJECT in compiler.1 compiler.2 goeasyconfig.1 ncdns.1 ncp11.1 ncprop279.1 plain-binaries.1 release.1; do PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) + if [[ "$PROJECT_BASE" == "compiler" ]]; then + if [[ "$OS" == "android" ]]; then + PROJECT_BASE=android-toolchain + fi + if [[ "$OS" == "linux" ]]; then + PROJECT_BASE=gcc + fi + if [[ "$OS" == "windows" ]]; then + PROJECT_BASE=mingw-w64 + fi + if [[ "$OS" == "osx" ]]; then + PROJECT_BASE=macosx-toolchain + fi + fi PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2) echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_docker_builder: timeout_in: 120m @@ -85,7 +99,7 @@ print_os_arch () { - \"./tools/cirrus_build_project.sh ${PROJECT_BASE} ${CHANNEL} ${OS} ${ARCH} 1\"" # Depend on previous project - if [[ "$PROJECT" == "goeasyconfig.1" ]]; then + if [[ "$PROJECT" == "compiler.1" ]]; then echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_download\"" else From 25ccedfb1b80f0f1db73b072ede03a2bcbf0c0cc Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 20:13:17 +0000 Subject: [PATCH 40/43] Cirrus: Split off macosx-toolchain output cache --- .cirrus.yml | 315 ++++++++++++++++++++++++++++++++++ tools/cirrus_build_project.sh | 23 +-- tools/cirrus_gen_yml.sh | 14 ++ 3 files changed, 335 insertions(+), 17 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index c3f2945..f676336 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -7,6 +7,13 @@ release_linux_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -41,6 +48,13 @@ release_linux_x86_64_gcc_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -80,6 +94,13 @@ release_linux_x86_64_gcc_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -119,6 +140,13 @@ release_linux_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -158,6 +186,13 @@ release_linux_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -197,6 +232,13 @@ release_linux_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -236,6 +278,13 @@ release_linux_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -275,6 +324,13 @@ release_linux_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -314,6 +370,13 @@ release_linux_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -353,6 +416,13 @@ release_linux_i686_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -387,6 +457,13 @@ release_linux_i686_gcc_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -426,6 +503,13 @@ release_linux_i686_gcc_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -465,6 +549,13 @@ release_linux_i686_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -504,6 +595,13 @@ release_linux_i686_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -543,6 +641,13 @@ release_linux_i686_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -582,6 +687,13 @@ release_linux_i686_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -621,6 +733,13 @@ release_linux_i686_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -660,6 +779,13 @@ release_linux_i686_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -699,6 +825,13 @@ release_windows_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -733,6 +866,13 @@ release_windows_x86_64_mingw-w64_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -772,6 +912,13 @@ release_windows_x86_64_mingw-w64_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -811,6 +958,13 @@ release_windows_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -850,6 +1004,13 @@ release_windows_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -889,6 +1050,13 @@ release_windows_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -928,6 +1096,13 @@ release_windows_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -967,6 +1142,13 @@ release_windows_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -1006,6 +1188,13 @@ release_windows_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -1045,6 +1234,13 @@ release_windows_i686_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1079,6 +1275,13 @@ release_windows_i686_mingw-w64_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1118,6 +1321,13 @@ release_windows_i686_mingw-w64_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1157,6 +1367,13 @@ release_windows_i686_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1196,6 +1413,13 @@ release_windows_i686_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1235,6 +1459,13 @@ release_windows_i686_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1274,6 +1505,13 @@ release_windows_i686_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1313,6 +1551,13 @@ release_windows_i686_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1352,6 +1597,13 @@ release_windows_i686_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1391,6 +1643,13 @@ release_osx_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1425,6 +1684,13 @@ release_osx_x86_64_macosx-toolchain_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1464,6 +1730,13 @@ release_osx_x86_64_macosx-toolchain_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1503,6 +1776,13 @@ release_osx_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1542,6 +1822,13 @@ release_osx_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1581,6 +1868,13 @@ release_osx_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1620,6 +1914,13 @@ release_osx_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1659,6 +1960,13 @@ release_osx_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1698,6 +2006,13 @@ release_osx_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index 27acdc4..fb10ae0 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -32,14 +32,8 @@ patch -p1 < ../tools/checkpoints.patch popd echo "Moving caches..." -if [[ -e "./fonts/.git" ]]; then - echo "git_clones/fonts was cached, moving it to the right place..." - mv ./fonts ./git_clones/fonts -else - echo "git_clones/fonts was not cached." - rm -rf ./fonts ./git_clones/fonts -fi -ls ./git_clones +mv ./out_cache1/* ./out/ || true +rm -rf ./out_cache1/* || true echo "Unpacking interrupted cache..." ./tools/cirrus_unpack_interrupted.sh || true @@ -77,15 +71,6 @@ else rm -rf ./tmp/interrupted_dirs/* || true fi -echo "Moving caches..." -if [[ -e "git_clones/fonts" ]]; then - echo "git_clones/fonts is ready to be cached, moving it to the right place..." - mv git_clones/fonts ./ -else - echo "git_clones/fonts is not ready to be not cached." - mkdir -p ./fonts -fi - # The cache has a size limit, so we need to clean useless data from it. The # container-images are very large and seem to be fairly harmless to remove. # Maybe later if we have more pressure to shrink, we could remove the @@ -93,5 +78,9 @@ fi echo "Cleaning cache..." rm -rfv out/container-image +echo "Moving caches..." +mv ./out/macosx-toolchain ./out_cache1/ || true +rm -rf ./out/macosx-toolchain || true + echo "Packing interrupted cache..." ./tools/cirrus_pack_interrupted.sh || true diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 56fbb15..c7822e9 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -17,6 +17,13 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p out\" + out1_${CHANNEL}_${OS}_${ARCH}_cache: + folder: out_cache1 + fingerprint_script: + - \"echo out1_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p out_cache1\" git_${CHANNEL}_${OS}_${ARCH}_cache: folder: git_clones fingerprint_script: @@ -70,6 +77,13 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p out\" + out1_${CHANNEL}_${OS}_${ARCH}_cache: + folder: out_cache1 + fingerprint_script: + - \"echo out1_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p out_cache1\" git_${CHANNEL}_${OS}_${ARCH}_cache: folder: git_clones fingerprint_script: From 1dda171af37b96882bfc3a0fa06caaa6f85681be Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sun, 21 Feb 2021 01:49:34 +0000 Subject: [PATCH 41/43] Cirrus: Make output cache a bit more robust --- tools/cirrus_build_project.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index fb10ae0..e329e5e 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -16,7 +16,7 @@ lscpu free -m echo "Installing rbm deps..." -APT_DEPS="libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc" +APT_DEPS="libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc rsync" apt-get install -y $APT_DEPS || (sleep 15s && apt-get install -y $APT_DEPS) echo "Pulling rbm..." @@ -31,9 +31,8 @@ pushd tor-browser-build patch -p1 < ../tools/checkpoints.patch popd -echo "Moving caches..." -mv ./out_cache1/* ./out/ || true -rm -rf ./out_cache1/* || true +echo "Restoring caches..." +cp -a ./out_cache1/* ./out/ || true echo "Unpacking interrupted cache..." ./tools/cirrus_unpack_interrupted.sh || true @@ -78,8 +77,8 @@ fi echo "Cleaning cache..." rm -rfv out/container-image -echo "Moving caches..." -mv ./out/macosx-toolchain ./out_cache1/ || true +echo "Splitting caches..." +rsync -avu --delete ./out/macosx-toolchain ./out_cache1/ rm -rf ./out/macosx-toolchain || true echo "Packing interrupted cache..." From 050d0e4443a0e6b528e3aec37a7a70655293ebb8 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sun, 21 Feb 2021 02:15:05 +0000 Subject: [PATCH 42/43] Cirrus: Run "make clean" dry-run --- tools/cirrus_build_project.sh | 5 +++++ tools/rbm.local.conf.onetarget | 14 ++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 tools/rbm.local.conf.onetarget diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index e329e5e..59e8956 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -25,6 +25,8 @@ make submodule-update echo "Configuring rbm..." # Print logs to Cirrus. cat rbm.local.conf.example | sed "s/#build_log: '-'/build_log: '-'/g" > rbm.local.conf +# Configure "make clean" +cat tools/rbm.local.conf.onetarget | sed "s/CHANNEL/$CHANNEL/g" | sed "s/ncdns-all/ncdns-$OS-$ARCH/g" >> rbm.local.conf echo "Patching rbm..." pushd tor-browser-build @@ -76,6 +78,9 @@ fi # debootstrap-images too. echo "Cleaning cache..." rm -rfv out/container-image +if [[ "$SHOULD_BUILD" -eq 0 ]]; then + ./tools/clean-old --dry-run +fi echo "Splitting caches..." rsync -avu --delete ./out/macosx-toolchain ./out_cache1/ diff --git a/tools/rbm.local.conf.onetarget b/tools/rbm.local.conf.onetarget new file mode 100644 index 0000000..d349b4d --- /dev/null +++ b/tools/rbm.local.conf.onetarget @@ -0,0 +1,14 @@ + +var: + ### The clean configuration is used by the cleaning script to find the + ### branches and build targets you are using, to compute the list of + ### files that should be kept. + ### + ### If you only do alpha builds for all platforms, you can use the + ### following configuration: + clean: + HEAD: + - project: release + target: + - CHANNEL + - ncdns-all From 2a986230f4b4f81bb0fea0641947f0267954707e Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sun, 21 Feb 2021 02:16:29 +0000 Subject: [PATCH 43/43] Cirrus: Allow rsync to fail --- tools/cirrus_build_project.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index 59e8956..b8dd172 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -83,7 +83,7 @@ if [[ "$SHOULD_BUILD" -eq 0 ]]; then fi echo "Splitting caches..." -rsync -avu --delete ./out/macosx-toolchain ./out_cache1/ +rsync -avu --delete ./out/macosx-toolchain ./out_cache1/ || true rm -rf ./out/macosx-toolchain || true echo "Packing interrupted cache..."