From 5d67272dcafa06890eb8205af9f9472ffe61ecb6 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Mon, 21 Mar 2022 14:02:45 +0000 Subject: [PATCH] Cirrus: Build compiler with 8 threads Use 1 thread for all other projects. Disable the checkpointing patch to see if 8 threads is enough to build without it. --- .cirrus.yml | 749 +++++++++++++++------------------ tools/cirrus_gen_yml.sh | 42 +- tools/patch-tor-to-namecoin.sh | 6 +- 3 files changed, 372 insertions(+), 425 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 46d2d23..39f35eb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,12 @@ # This file is auto-generated by tools/cirrus_gen_yml.sh -- do not edit manually! -release_linux_x86_64_download_docker_builder: +release_linux_x86_64_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -64,8 +70,15 @@ release_linux_x86_64_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_linux_x86_64_gcc_1_docker_builder: +release_linux_x86_64_gcc_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -136,84 +149,17 @@ release_linux_x86_64_gcc_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_linux_x86_64_download" -release_linux_x86_64_gcc_2_docker_builder: - timeout_in: 120m - out_release_linux_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_linux_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_linux_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_linux_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_linux_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_linux_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_linux_x86_64" - reupload_on_changes: true - interrupted_ab_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_linux_x86_64" - reupload_on_changes: true - interrupted_ac_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_linux_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh gcc release linux x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_linux_x86_64_gcc_1" - -release_linux_x86_64_goeasyconfig_1_docker_builder: +release_linux_x86_64_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -284,10 +230,17 @@ release_linux_x86_64_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_linux_x86_64_gcc_2" + - "release_linux_x86_64_gcc_para1" -release_linux_x86_64_ncdns_1_docker_builder: +release_linux_x86_64_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -358,10 +311,17 @@ release_linux_x86_64_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_goeasyconfig_1" -release_linux_x86_64_ncprop279_1_docker_builder: +release_linux_x86_64_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -432,10 +392,17 @@ release_linux_x86_64_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_ncdns_1" -release_linux_x86_64_plain-binaries_1_docker_builder: +release_linux_x86_64_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -506,10 +473,17 @@ release_linux_x86_64_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_ncprop279_1" -release_linux_x86_64_release_nosign_docker_builder: +release_linux_x86_64_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -582,11 +556,18 @@ release_linux_x86_64_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_linux_x86_64_plain-binaries_1" -release_linux_x86_64_release_sign_docker_builder: +release_linux_x86_64_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -662,10 +643,17 @@ release_linux_x86_64_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_plain-binaries_1" -release_linux_i686_download_docker_builder: +release_linux_i686_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -729,8 +717,15 @@ release_linux_i686_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_linux_i686_gcc_1_docker_builder: +release_linux_i686_gcc_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -801,84 +796,17 @@ release_linux_i686_gcc_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_linux_i686_download" -release_linux_i686_gcc_2_docker_builder: - timeout_in: 120m - out_release_linux_i686_cache: - folder: out - fingerprint_script: - - "echo out_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_linux_i686_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_linux_i686_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_linux_i686_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_linux_i686_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_linux_i686_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_linux_i686" - reupload_on_changes: true - interrupted_ab_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_linux_i686" - reupload_on_changes: true - interrupted_ac_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_linux_i686" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh gcc release linux i686 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_linux_i686_gcc_1" - -release_linux_i686_goeasyconfig_1_docker_builder: +release_linux_i686_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -949,10 +877,17 @@ release_linux_i686_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_linux_i686_gcc_2" + - "release_linux_i686_gcc_para1" -release_linux_i686_ncdns_1_docker_builder: +release_linux_i686_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1023,10 +958,17 @@ release_linux_i686_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_goeasyconfig_1" -release_linux_i686_ncprop279_1_docker_builder: +release_linux_i686_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1097,10 +1039,17 @@ release_linux_i686_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_ncdns_1" -release_linux_i686_plain-binaries_1_docker_builder: +release_linux_i686_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1171,10 +1120,17 @@ release_linux_i686_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_ncprop279_1" -release_linux_i686_release_nosign_docker_builder: +release_linux_i686_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1247,11 +1203,18 @@ release_linux_i686_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_linux_i686_plain-binaries_1" -release_linux_i686_release_sign_docker_builder: +release_linux_i686_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1327,10 +1290,17 @@ release_linux_i686_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_plain-binaries_1" -release_windows_x86_64_download_docker_builder: +release_windows_x86_64_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1394,8 +1364,15 @@ release_windows_x86_64_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_windows_x86_64_mingw-w64_1_docker_builder: +release_windows_x86_64_mingw-w64_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1466,84 +1443,17 @@ release_windows_x86_64_mingw-w64_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_windows_x86_64_download" -release_windows_x86_64_mingw-w64_2_docker_builder: - timeout_in: 120m - out_release_windows_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_windows_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_windows_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_windows_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_windows_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_windows_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_windows_x86_64" - reupload_on_changes: true - interrupted_ab_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_windows_x86_64" - reupload_on_changes: true - interrupted_ac_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_windows_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh mingw-w64 release windows x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_windows_x86_64_mingw-w64_1" - -release_windows_x86_64_goeasyconfig_1_docker_builder: +release_windows_x86_64_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1614,10 +1524,17 @@ release_windows_x86_64_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_windows_x86_64_mingw-w64_2" + - "release_windows_x86_64_mingw-w64_para1" -release_windows_x86_64_ncdns_1_docker_builder: +release_windows_x86_64_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1688,10 +1605,17 @@ release_windows_x86_64_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_goeasyconfig_1" -release_windows_x86_64_ncprop279_1_docker_builder: +release_windows_x86_64_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1762,10 +1686,17 @@ release_windows_x86_64_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_ncdns_1" -release_windows_x86_64_plain-binaries_1_docker_builder: +release_windows_x86_64_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1836,10 +1767,17 @@ release_windows_x86_64_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_ncprop279_1" -release_windows_x86_64_release_nosign_docker_builder: +release_windows_x86_64_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1912,11 +1850,18 @@ release_windows_x86_64_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_windows_x86_64_plain-binaries_1" -release_windows_x86_64_release_sign_docker_builder: +release_windows_x86_64_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1992,10 +1937,17 @@ release_windows_x86_64_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_plain-binaries_1" -release_windows_i686_download_docker_builder: +release_windows_i686_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2059,8 +2011,15 @@ release_windows_i686_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_windows_i686_mingw-w64_1_docker_builder: +release_windows_i686_mingw-w64_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2131,84 +2090,17 @@ release_windows_i686_mingw-w64_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_windows_i686_download" -release_windows_i686_mingw-w64_2_docker_builder: - timeout_in: 120m - out_release_windows_i686_cache: - folder: out - fingerprint_script: - - "echo out_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_windows_i686_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_windows_i686_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_windows_i686_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_windows_i686_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_windows_i686_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_windows_i686" - reupload_on_changes: true - interrupted_ab_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_windows_i686" - reupload_on_changes: true - interrupted_ac_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_windows_i686" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh mingw-w64 release windows i686 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_windows_i686_mingw-w64_1" - -release_windows_i686_goeasyconfig_1_docker_builder: +release_windows_i686_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2279,10 +2171,17 @@ release_windows_i686_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_windows_i686_mingw-w64_2" + - "release_windows_i686_mingw-w64_para1" -release_windows_i686_ncdns_1_docker_builder: +release_windows_i686_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2353,10 +2252,17 @@ release_windows_i686_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_goeasyconfig_1" -release_windows_i686_ncprop279_1_docker_builder: +release_windows_i686_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2427,10 +2333,17 @@ release_windows_i686_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_ncdns_1" -release_windows_i686_plain-binaries_1_docker_builder: +release_windows_i686_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2501,10 +2414,17 @@ release_windows_i686_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_ncprop279_1" -release_windows_i686_release_nosign_docker_builder: +release_windows_i686_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2577,11 +2497,18 @@ release_windows_i686_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_windows_i686_plain-binaries_1" -release_windows_i686_release_sign_docker_builder: +release_windows_i686_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2657,10 +2584,17 @@ release_windows_i686_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_plain-binaries_1" -release_osx_x86_64_download_docker_builder: +release_osx_x86_64_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2724,8 +2658,15 @@ release_osx_x86_64_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_osx_x86_64_macosx-toolchain_1_docker_builder: +release_osx_x86_64_macosx-toolchain_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2796,84 +2737,17 @@ release_osx_x86_64_macosx-toolchain_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_osx_x86_64_download" -release_osx_x86_64_macosx-toolchain_2_docker_builder: - timeout_in: 120m - out_release_osx_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_osx_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_osx_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_osx_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_osx_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_osx_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_osx_x86_64" - reupload_on_changes: true - interrupted_ab_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_osx_x86_64" - reupload_on_changes: true - interrupted_ac_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_osx_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh macosx-toolchain release osx x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_osx_x86_64_macosx-toolchain_1" - -release_osx_x86_64_goeasyconfig_1_docker_builder: +release_osx_x86_64_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2944,10 +2818,17 @@ release_osx_x86_64_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_osx_x86_64_macosx-toolchain_2" + - "release_osx_x86_64_macosx-toolchain_para1" -release_osx_x86_64_ncdns_1_docker_builder: +release_osx_x86_64_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3018,10 +2899,17 @@ release_osx_x86_64_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_goeasyconfig_1" -release_osx_x86_64_ncprop279_1_docker_builder: +release_osx_x86_64_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3092,10 +2980,17 @@ release_osx_x86_64_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_ncdns_1" -release_osx_x86_64_plain-binaries_1_docker_builder: +release_osx_x86_64_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3166,10 +3061,17 @@ release_osx_x86_64_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_ncprop279_1" -release_osx_x86_64_release_nosign_docker_builder: +release_osx_x86_64_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3242,11 +3144,18 @@ release_osx_x86_64_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_osx_x86_64_plain-binaries_1" -release_osx_x86_64_release_sign_docker_builder: +release_osx_x86_64_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3322,14 +3231,22 @@ release_osx_x86_64_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_plain-binaries_1" -bump_docker_builder: +bump_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G bump_script: - "./tools/cirrus_build_project.sh null null null null 0" env: BUMP_DEPS: 1 + RBM_NUM_PROCS: 1 DEPLOY_KEY: ENCRYPTED[7969cc42abbc36c75c5673f2227e2eeec92577391c28c678243f95e01edffa17137e52cbddbe3e409cdab78a637edec5] PR_TOKEN: ENCRYPTED[91c45714bbbcf5b1fbb124475368332fcec4020258c5c4316ea9d07e3933982c6d179b925d6f7488978528ca99a737f3] only_if: $CIRRUS_REPO_OWNER == "namecoin" && $CIRRUS_PR == "" diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 6373e5a..a681bc6 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -8,7 +8,13 @@ print_os_arch () { local ARCH="$2" # Pre-download tarballs and Git repos - echo "${CHANNEL}_${OS}_${ARCH}_download_docker_builder: + echo "${CHANNEL}_${OS}_${ARCH}_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -71,11 +77,13 @@ print_os_arch () { - \"./tools/cirrus_build_project.sh plain-binaries ${CHANNEL} ${OS} ${ARCH} 0\"" echo " env: CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0" + BUMP_DEPS: 0 + RBM_NUM_PROCS: 1" echo "" # TODO fine-tune this list - for PROJECT in compiler.1 compiler.2 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do + # Use "para" prefix to run with 8 threads; otherwise will use 1 thread. + for PROJECT in compiler.para1 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) if [[ "$PROJECT_BASE" == "compiler" ]]; then if [[ "$OS" == "android" ]]; then @@ -92,7 +100,17 @@ print_os_arch () { fi fi PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2) - echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_docker_builder: + PARA_THREADS=1 + if echo $PROJECT_ITER | grep -q para ; then + PARA_THREADS=8 + fi + echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: ${PARA_THREADS} + memory: 16G timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -176,13 +194,14 @@ print_os_arch () { fi echo " env: CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0" + BUMP_DEPS: 0 + RBM_NUM_PROCS: $PARA_THREADS" if [[ "$PROJECT_ITER" == "nosign" ]]; then echo ' only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != ""' fi # Depend on previous project - if [[ "$PROJECT" == "compiler.1" ]]; then + if [[ "$PROJECT" == "compiler.para1" ]]; then echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_download\"" else @@ -209,11 +228,18 @@ for CHANNEL in release; do print_os_arch osx x86_64 done -echo 'bump_docker_builder: +echo 'bump_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G bump_script: - "./tools/cirrus_build_project.sh null null null null 0" env: BUMP_DEPS: 1 + RBM_NUM_PROCS: 1 DEPLOY_KEY: ENCRYPTED[7969cc42abbc36c75c5673f2227e2eeec92577391c28c678243f95e01edffa17137e52cbddbe3e409cdab78a637edec5] PR_TOKEN: ENCRYPTED[91c45714bbbcf5b1fbb124475368332fcec4020258c5c4316ea9d07e3933982c6d179b925d6f7488978528ca99a737f3] only_if: $CIRRUS_REPO_OWNER == "namecoin" && $CIRRUS_PR == ""' @@ -223,4 +249,4 @@ echo 'bump_docker_builder: # Might want to increase the timeout -- but we're already using the 2 hour max. # Might want to bump the CPU count -- but that's blocked by cirrus-ci-docs issue #741. # Might want to split into smaller project sets. -# What is the CPU count limit? "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 16.0 CPU. docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS). +# What is the CPU count limit? "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 8 CPU for a single task. docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS). diff --git a/tools/patch-tor-to-namecoin.sh b/tools/patch-tor-to-namecoin.sh index 72e2194..640b741 100755 --- a/tools/patch-tor-to-namecoin.sh +++ b/tools/patch-tor-to-namecoin.sh @@ -7,7 +7,11 @@ pushd tor-browser-build # Undo any of Namecoin's patches that might have been done previously. git reset --hard HEAD git clean -dfx -patch -p1 < ../tools/checkpoints.patch + +# The checkpoint patch is disabled while we evaluate whether the move to 8x +# CPU's per VM is sufficient to build within a single task time limit. +#patch -p1 < ../tools/checkpoints.patch + popd # Rename torbrowser to ncdns