From 5d67272dcafa06890eb8205af9f9472ffe61ecb6 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Mon, 21 Mar 2022 14:02:45 +0000 Subject: [PATCH 1/4] Cirrus: Build compiler with 8 threads Use 1 thread for all other projects. Disable the checkpointing patch to see if 8 threads is enough to build without it. --- .cirrus.yml | 749 +++++++++++++++------------------ tools/cirrus_gen_yml.sh | 42 +- tools/patch-tor-to-namecoin.sh | 6 +- 3 files changed, 372 insertions(+), 425 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 46d2d23..39f35eb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,12 @@ # This file is auto-generated by tools/cirrus_gen_yml.sh -- do not edit manually! -release_linux_x86_64_download_docker_builder: +release_linux_x86_64_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -64,8 +70,15 @@ release_linux_x86_64_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_linux_x86_64_gcc_1_docker_builder: +release_linux_x86_64_gcc_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -136,84 +149,17 @@ release_linux_x86_64_gcc_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_linux_x86_64_download" -release_linux_x86_64_gcc_2_docker_builder: - timeout_in: 120m - out_release_linux_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_linux_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_linux_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_linux_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_linux_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_linux_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_linux_x86_64" - reupload_on_changes: true - interrupted_ab_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_linux_x86_64" - reupload_on_changes: true - interrupted_ac_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_linux_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh gcc release linux x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_linux_x86_64_gcc_1" - -release_linux_x86_64_goeasyconfig_1_docker_builder: +release_linux_x86_64_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -284,10 +230,17 @@ release_linux_x86_64_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_linux_x86_64_gcc_2" + - "release_linux_x86_64_gcc_para1" -release_linux_x86_64_ncdns_1_docker_builder: +release_linux_x86_64_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -358,10 +311,17 @@ release_linux_x86_64_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_goeasyconfig_1" -release_linux_x86_64_ncprop279_1_docker_builder: +release_linux_x86_64_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -432,10 +392,17 @@ release_linux_x86_64_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_ncdns_1" -release_linux_x86_64_plain-binaries_1_docker_builder: +release_linux_x86_64_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -506,10 +473,17 @@ release_linux_x86_64_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_ncprop279_1" -release_linux_x86_64_release_nosign_docker_builder: +release_linux_x86_64_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -582,11 +556,18 @@ release_linux_x86_64_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_linux_x86_64_plain-binaries_1" -release_linux_x86_64_release_sign_docker_builder: +release_linux_x86_64_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -662,10 +643,17 @@ release_linux_x86_64_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_x86_64_plain-binaries_1" -release_linux_i686_download_docker_builder: +release_linux_i686_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -729,8 +717,15 @@ release_linux_i686_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_linux_i686_gcc_1_docker_builder: +release_linux_i686_gcc_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -801,84 +796,17 @@ release_linux_i686_gcc_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_linux_i686_download" -release_linux_i686_gcc_2_docker_builder: - timeout_in: 120m - out_release_linux_i686_cache: - folder: out - fingerprint_script: - - "echo out_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_linux_i686_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_linux_i686_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_linux_i686_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_linux_i686_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_linux_i686_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_linux_i686" - reupload_on_changes: true - interrupted_ab_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_linux_i686" - reupload_on_changes: true - interrupted_ac_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_linux_i686" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh gcc release linux i686 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_linux_i686_gcc_1" - -release_linux_i686_goeasyconfig_1_docker_builder: +release_linux_i686_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -949,10 +877,17 @@ release_linux_i686_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_linux_i686_gcc_2" + - "release_linux_i686_gcc_para1" -release_linux_i686_ncdns_1_docker_builder: +release_linux_i686_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1023,10 +958,17 @@ release_linux_i686_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_goeasyconfig_1" -release_linux_i686_ncprop279_1_docker_builder: +release_linux_i686_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1097,10 +1039,17 @@ release_linux_i686_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_ncdns_1" -release_linux_i686_plain-binaries_1_docker_builder: +release_linux_i686_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1171,10 +1120,17 @@ release_linux_i686_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_ncprop279_1" -release_linux_i686_release_nosign_docker_builder: +release_linux_i686_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1247,11 +1203,18 @@ release_linux_i686_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_linux_i686_plain-binaries_1" -release_linux_i686_release_sign_docker_builder: +release_linux_i686_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1327,10 +1290,17 @@ release_linux_i686_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_linux_i686_plain-binaries_1" -release_windows_x86_64_download_docker_builder: +release_windows_x86_64_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1394,8 +1364,15 @@ release_windows_x86_64_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_windows_x86_64_mingw-w64_1_docker_builder: +release_windows_x86_64_mingw-w64_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1466,84 +1443,17 @@ release_windows_x86_64_mingw-w64_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_windows_x86_64_download" -release_windows_x86_64_mingw-w64_2_docker_builder: - timeout_in: 120m - out_release_windows_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_windows_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_windows_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_windows_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_windows_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_windows_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_windows_x86_64" - reupload_on_changes: true - interrupted_ab_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_windows_x86_64" - reupload_on_changes: true - interrupted_ac_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_windows_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh mingw-w64 release windows x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_windows_x86_64_mingw-w64_1" - -release_windows_x86_64_goeasyconfig_1_docker_builder: +release_windows_x86_64_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1614,10 +1524,17 @@ release_windows_x86_64_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_windows_x86_64_mingw-w64_2" + - "release_windows_x86_64_mingw-w64_para1" -release_windows_x86_64_ncdns_1_docker_builder: +release_windows_x86_64_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1688,10 +1605,17 @@ release_windows_x86_64_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_goeasyconfig_1" -release_windows_x86_64_ncprop279_1_docker_builder: +release_windows_x86_64_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1762,10 +1686,17 @@ release_windows_x86_64_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_ncdns_1" -release_windows_x86_64_plain-binaries_1_docker_builder: +release_windows_x86_64_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1836,10 +1767,17 @@ release_windows_x86_64_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_ncprop279_1" -release_windows_x86_64_release_nosign_docker_builder: +release_windows_x86_64_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1912,11 +1850,18 @@ release_windows_x86_64_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_windows_x86_64_plain-binaries_1" -release_windows_x86_64_release_sign_docker_builder: +release_windows_x86_64_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1992,10 +1937,17 @@ release_windows_x86_64_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_x86_64_plain-binaries_1" -release_windows_i686_download_docker_builder: +release_windows_i686_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2059,8 +2011,15 @@ release_windows_i686_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_windows_i686_mingw-w64_1_docker_builder: +release_windows_i686_mingw-w64_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2131,84 +2090,17 @@ release_windows_i686_mingw-w64_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_windows_i686_download" -release_windows_i686_mingw-w64_2_docker_builder: - timeout_in: 120m - out_release_windows_i686_cache: - folder: out - fingerprint_script: - - "echo out_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_windows_i686_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_windows_i686_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_windows_i686_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_windows_i686_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_windows_i686_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_windows_i686" - reupload_on_changes: true - interrupted_ab_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_windows_i686" - reupload_on_changes: true - interrupted_ac_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_windows_i686" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh mingw-w64 release windows i686 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_windows_i686_mingw-w64_1" - -release_windows_i686_goeasyconfig_1_docker_builder: +release_windows_i686_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2279,10 +2171,17 @@ release_windows_i686_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_windows_i686_mingw-w64_2" + - "release_windows_i686_mingw-w64_para1" -release_windows_i686_ncdns_1_docker_builder: +release_windows_i686_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2353,10 +2252,17 @@ release_windows_i686_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_goeasyconfig_1" -release_windows_i686_ncprop279_1_docker_builder: +release_windows_i686_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2427,10 +2333,17 @@ release_windows_i686_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_ncdns_1" -release_windows_i686_plain-binaries_1_docker_builder: +release_windows_i686_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2501,10 +2414,17 @@ release_windows_i686_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_ncprop279_1" -release_windows_i686_release_nosign_docker_builder: +release_windows_i686_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2577,11 +2497,18 @@ release_windows_i686_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_windows_i686_plain-binaries_1" -release_windows_i686_release_sign_docker_builder: +release_windows_i686_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2657,10 +2584,17 @@ release_windows_i686_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_windows_i686_plain-binaries_1" -release_osx_x86_64_download_docker_builder: +release_osx_x86_64_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2724,8 +2658,15 @@ release_osx_x86_64_download_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 -release_osx_x86_64_macosx-toolchain_1_docker_builder: +release_osx_x86_64_macosx-toolchain_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2796,84 +2737,17 @@ release_osx_x86_64_macosx-toolchain_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 depends_on: - "release_osx_x86_64_download" -release_osx_x86_64_macosx-toolchain_2_docker_builder: - timeout_in: 120m - out_release_osx_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_osx_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_osx_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_osx_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_osx_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_osx_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_osx_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_osx_x86_64" - reupload_on_changes: true - interrupted_ab_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_osx_x86_64" - reupload_on_changes: true - interrupted_ac_release_osx_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_osx_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh macosx-toolchain release osx x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - depends_on: - - "release_osx_x86_64_macosx-toolchain_1" - -release_osx_x86_64_goeasyconfig_1_docker_builder: +release_osx_x86_64_goeasyconfig_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2944,10 +2818,17 @@ release_osx_x86_64_goeasyconfig_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - - "release_osx_x86_64_macosx-toolchain_2" + - "release_osx_x86_64_macosx-toolchain_para1" -release_osx_x86_64_ncdns_1_docker_builder: +release_osx_x86_64_ncdns_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3018,10 +2899,17 @@ release_osx_x86_64_ncdns_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_goeasyconfig_1" -release_osx_x86_64_ncprop279_1_docker_builder: +release_osx_x86_64_ncprop279_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3092,10 +2980,17 @@ release_osx_x86_64_ncprop279_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_ncdns_1" -release_osx_x86_64_plain-binaries_1_docker_builder: +release_osx_x86_64_plain-binaries_1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3166,10 +3061,17 @@ release_osx_x86_64_plain-binaries_1_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_ncprop279_1" -release_osx_x86_64_release_nosign_docker_builder: +release_osx_x86_64_release_nosign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3242,11 +3144,18 @@ release_osx_x86_64_release_nosign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != "" depends_on: - "release_osx_x86_64_plain-binaries_1" -release_osx_x86_64_release_sign_docker_builder: +release_osx_x86_64_release_sign_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3322,14 +3231,22 @@ release_osx_x86_64_release_sign_docker_builder: env: CIRRUS_LOG_TIMESTAMP: true BUMP_DEPS: 0 + RBM_NUM_PROCS: 1 depends_on: - "release_osx_x86_64_plain-binaries_1" -bump_docker_builder: +bump_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G bump_script: - "./tools/cirrus_build_project.sh null null null null 0" env: BUMP_DEPS: 1 + RBM_NUM_PROCS: 1 DEPLOY_KEY: ENCRYPTED[7969cc42abbc36c75c5673f2227e2eeec92577391c28c678243f95e01edffa17137e52cbddbe3e409cdab78a637edec5] PR_TOKEN: ENCRYPTED[91c45714bbbcf5b1fbb124475368332fcec4020258c5c4316ea9d07e3933982c6d179b925d6f7488978528ca99a737f3] only_if: $CIRRUS_REPO_OWNER == "namecoin" && $CIRRUS_PR == "" diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 6373e5a..a681bc6 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -8,7 +8,13 @@ print_os_arch () { local ARCH="$2" # Pre-download tarballs and Git repos - echo "${CHANNEL}_${OS}_${ARCH}_download_docker_builder: + echo "${CHANNEL}_${OS}_${ARCH}_download_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -71,11 +77,13 @@ print_os_arch () { - \"./tools/cirrus_build_project.sh plain-binaries ${CHANNEL} ${OS} ${ARCH} 0\"" echo " env: CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0" + BUMP_DEPS: 0 + RBM_NUM_PROCS: 1" echo "" # TODO fine-tune this list - for PROJECT in compiler.1 compiler.2 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do + # Use "para" prefix to run with 8 threads; otherwise will use 1 thread. + for PROJECT in compiler.para1 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) if [[ "$PROJECT_BASE" == "compiler" ]]; then if [[ "$OS" == "android" ]]; then @@ -92,7 +100,17 @@ print_os_arch () { fi fi PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2) - echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_docker_builder: + PARA_THREADS=1 + if echo $PROJECT_ITER | grep -q para ; then + PARA_THREADS=8 + fi + echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: ${PARA_THREADS} + memory: 16G timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -176,13 +194,14 @@ print_os_arch () { fi echo " env: CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0" + BUMP_DEPS: 0 + RBM_NUM_PROCS: $PARA_THREADS" if [[ "$PROJECT_ITER" == "nosign" ]]; then echo ' only_if: $CIRRUS_REPO_OWNER != "namecoin" || "$CIRRUS_PR" != ""' fi # Depend on previous project - if [[ "$PROJECT" == "compiler.1" ]]; then + if [[ "$PROJECT" == "compiler.para1" ]]; then echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_download\"" else @@ -209,11 +228,18 @@ for CHANNEL in release; do print_os_arch osx x86_64 done -echo 'bump_docker_builder: +echo 'bump_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 1 + memory: 16G bump_script: - "./tools/cirrus_build_project.sh null null null null 0" env: BUMP_DEPS: 1 + RBM_NUM_PROCS: 1 DEPLOY_KEY: ENCRYPTED[7969cc42abbc36c75c5673f2227e2eeec92577391c28c678243f95e01edffa17137e52cbddbe3e409cdab78a637edec5] PR_TOKEN: ENCRYPTED[91c45714bbbcf5b1fbb124475368332fcec4020258c5c4316ea9d07e3933982c6d179b925d6f7488978528ca99a737f3] only_if: $CIRRUS_REPO_OWNER == "namecoin" && $CIRRUS_PR == ""' @@ -223,4 +249,4 @@ echo 'bump_docker_builder: # Might want to increase the timeout -- but we're already using the 2 hour max. # Might want to bump the CPU count -- but that's blocked by cirrus-ci-docs issue #741. # Might want to split into smaller project sets. -# What is the CPU count limit? "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 16.0 CPU. docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS). +# What is the CPU count limit? "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 8 CPU for a single task. docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS). diff --git a/tools/patch-tor-to-namecoin.sh b/tools/patch-tor-to-namecoin.sh index 72e2194..640b741 100755 --- a/tools/patch-tor-to-namecoin.sh +++ b/tools/patch-tor-to-namecoin.sh @@ -7,7 +7,11 @@ pushd tor-browser-build # Undo any of Namecoin's patches that might have been done previously. git reset --hard HEAD git clean -dfx -patch -p1 < ../tools/checkpoints.patch + +# The checkpoint patch is disabled while we evaluate whether the move to 8x +# CPU's per VM is sufficient to build within a single task time limit. +#patch -p1 < ../tools/checkpoints.patch + popd # Rename torbrowser to ncdns From db1cecb31736d82555acd99c82bed99739d47bf0 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Mon, 21 Mar 2022 14:11:03 +0000 Subject: [PATCH 2/4] Cirrus: Decrease 1-CPU VM's to 3 GiB RAM --- .cirrus.yml | 72 ++++++++++++++++++++--------------------- tools/cirrus_gen_yml.sh | 8 +++-- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 39f35eb..91fbb7a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -6,7 +6,7 @@ release_linux_x86_64_download_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -159,7 +159,7 @@ release_linux_x86_64_goeasyconfig_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -240,7 +240,7 @@ release_linux_x86_64_ncdns_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -321,7 +321,7 @@ release_linux_x86_64_ncprop279_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -402,7 +402,7 @@ release_linux_x86_64_plain-binaries_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -483,7 +483,7 @@ release_linux_x86_64_release_nosign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -567,7 +567,7 @@ release_linux_x86_64_release_sign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_x86_64_cache: folder: out @@ -653,7 +653,7 @@ release_linux_i686_download_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -806,7 +806,7 @@ release_linux_i686_goeasyconfig_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -887,7 +887,7 @@ release_linux_i686_ncdns_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -968,7 +968,7 @@ release_linux_i686_ncprop279_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1049,7 +1049,7 @@ release_linux_i686_plain-binaries_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1130,7 +1130,7 @@ release_linux_i686_release_nosign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1214,7 +1214,7 @@ release_linux_i686_release_sign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_linux_i686_cache: folder: out @@ -1300,7 +1300,7 @@ release_windows_x86_64_download_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1453,7 +1453,7 @@ release_windows_x86_64_goeasyconfig_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1534,7 +1534,7 @@ release_windows_x86_64_ncdns_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1615,7 +1615,7 @@ release_windows_x86_64_ncprop279_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1696,7 +1696,7 @@ release_windows_x86_64_plain-binaries_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1777,7 +1777,7 @@ release_windows_x86_64_release_nosign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1861,7 +1861,7 @@ release_windows_x86_64_release_sign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_x86_64_cache: folder: out @@ -1947,7 +1947,7 @@ release_windows_i686_download_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2100,7 +2100,7 @@ release_windows_i686_goeasyconfig_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2181,7 +2181,7 @@ release_windows_i686_ncdns_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2262,7 +2262,7 @@ release_windows_i686_ncprop279_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2343,7 +2343,7 @@ release_windows_i686_plain-binaries_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2424,7 +2424,7 @@ release_windows_i686_release_nosign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2508,7 +2508,7 @@ release_windows_i686_release_sign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_windows_i686_cache: folder: out @@ -2594,7 +2594,7 @@ release_osx_x86_64_download_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2747,7 +2747,7 @@ release_osx_x86_64_goeasyconfig_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2828,7 +2828,7 @@ release_osx_x86_64_ncdns_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2909,7 +2909,7 @@ release_osx_x86_64_ncprop279_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -2990,7 +2990,7 @@ release_osx_x86_64_plain-binaries_1_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3071,7 +3071,7 @@ release_osx_x86_64_release_nosign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3155,7 +3155,7 @@ release_osx_x86_64_release_sign_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_release_osx_x86_64_cache: folder: out @@ -3241,7 +3241,7 @@ bump_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G bump_script: - "./tools/cirrus_build_project.sh null null null null 0" env: diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index a681bc6..b42f5da 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -14,7 +14,7 @@ print_os_arch () { image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -101,8 +101,10 @@ print_os_arch () { fi PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2) PARA_THREADS=1 + PARA_RAM=3 if echo $PROJECT_ITER | grep -q para ; then PARA_THREADS=8 + PARA_RAM=16 fi echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_task: compute_engine_instance: @@ -110,7 +112,7 @@ print_os_arch () { image: family/docker-builder platform: linux cpu: ${PARA_THREADS} - memory: 16G + memory: ${PARA_RAM}G timeout_in: 120m out_${CHANNEL}_${OS}_${ARCH}_cache: folder: out @@ -234,7 +236,7 @@ echo 'bump_task: image: family/docker-builder platform: linux cpu: 1 - memory: 16G + memory: 3G bump_script: - "./tools/cirrus_build_project.sh null null null null 0" env: From 4c11673ebb3dca1cbb474e3a3f87b1db047c6149 Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Mon, 21 Mar 2022 17:20:17 +0000 Subject: [PATCH 3/4] Cirrus: Factor out clang project Should improve the time limit safety margin of the masosx-toolchain project. --- .cirrus.yml | 415 +++++++++++++++++++++++++++++++++++++++- tools/cirrus_gen_yml.sh | 4 +- 2 files changed, 412 insertions(+), 7 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 91fbb7a..0426d56 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -72,6 +72,87 @@ release_linux_x86_64_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 +release_linux_x86_64_clang_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G + timeout_in: 120m + out_release_linux_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" + out2_release_linux_x86_64_cache: + folder: out_cache2 + fingerprint_script: + - "echo out2_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache2" + out3_release_linux_x86_64_cache: + folder: out_cache3 + fingerprint_script: + - "echo out3_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache3" + git_release_linux_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + gitindex_release_linux_x86_64_cache: + folder: git_index + fingerprint_script: + - "echo gitindex_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_index" + interrupted_aa_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_linux_x86_64" + reupload_on_changes: true + interrupted_ab_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_x86_64" + reupload_on_changes: true + interrupted_ac_release_linux_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_x86_64" + reupload_on_changes: true + checkpoint_background_script: + # 110m caused the 2hr task timeout to be hit while the cache was uploading + # for macosx-toolchain, which broke subsequent builds. + - sleep 105m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh clang release linux x86_64 1" + env: + SIGN_BUILD: 0 + env: + CIRRUS_LOG_TIMESTAMP: true + BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 + depends_on: + - "release_linux_x86_64_download" + release_linux_x86_64_gcc_para1_task: compute_engine_instance: image_project: cirrus-images @@ -151,7 +232,7 @@ release_linux_x86_64_gcc_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_linux_x86_64_download" + - "release_linux_x86_64_clang_para1" release_linux_x86_64_goeasyconfig_1_task: compute_engine_instance: @@ -719,6 +800,87 @@ release_linux_i686_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 +release_linux_i686_clang_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G + timeout_in: 120m + out_release_linux_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" + out2_release_linux_i686_cache: + folder: out_cache2 + fingerprint_script: + - "echo out2_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache2" + out3_release_linux_i686_cache: + folder: out_cache3 + fingerprint_script: + - "echo out3_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache3" + git_release_linux_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + gitindex_release_linux_i686_cache: + folder: git_index + fingerprint_script: + - "echo gitindex_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_index" + interrupted_aa_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_linux_i686" + reupload_on_changes: true + interrupted_ab_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_linux_i686" + reupload_on_changes: true + interrupted_ac_release_linux_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_linux_i686" + reupload_on_changes: true + checkpoint_background_script: + # 110m caused the 2hr task timeout to be hit while the cache was uploading + # for macosx-toolchain, which broke subsequent builds. + - sleep 105m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh clang release linux i686 1" + env: + SIGN_BUILD: 0 + env: + CIRRUS_LOG_TIMESTAMP: true + BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 + depends_on: + - "release_linux_i686_download" + release_linux_i686_gcc_para1_task: compute_engine_instance: image_project: cirrus-images @@ -798,7 +960,7 @@ release_linux_i686_gcc_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_linux_i686_download" + - "release_linux_i686_clang_para1" release_linux_i686_goeasyconfig_1_task: compute_engine_instance: @@ -1366,6 +1528,87 @@ release_windows_x86_64_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 +release_windows_x86_64_clang_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G + timeout_in: 120m + out_release_windows_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" + out2_release_windows_x86_64_cache: + folder: out_cache2 + fingerprint_script: + - "echo out2_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache2" + out3_release_windows_x86_64_cache: + folder: out_cache3 + fingerprint_script: + - "echo out3_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache3" + git_release_windows_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + gitindex_release_windows_x86_64_cache: + folder: git_index + fingerprint_script: + - "echo gitindex_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_index" + interrupted_aa_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_windows_x86_64" + reupload_on_changes: true + interrupted_ab_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_x86_64" + reupload_on_changes: true + interrupted_ac_release_windows_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_x86_64" + reupload_on_changes: true + checkpoint_background_script: + # 110m caused the 2hr task timeout to be hit while the cache was uploading + # for macosx-toolchain, which broke subsequent builds. + - sleep 105m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh clang release windows x86_64 1" + env: + SIGN_BUILD: 0 + env: + CIRRUS_LOG_TIMESTAMP: true + BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 + depends_on: + - "release_windows_x86_64_download" + release_windows_x86_64_mingw-w64_para1_task: compute_engine_instance: image_project: cirrus-images @@ -1445,7 +1688,7 @@ release_windows_x86_64_mingw-w64_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_windows_x86_64_download" + - "release_windows_x86_64_clang_para1" release_windows_x86_64_goeasyconfig_1_task: compute_engine_instance: @@ -2013,6 +2256,87 @@ release_windows_i686_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 +release_windows_i686_clang_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G + timeout_in: 120m + out_release_windows_i686_cache: + folder: out + fingerprint_script: + - "echo out_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" + out2_release_windows_i686_cache: + folder: out_cache2 + fingerprint_script: + - "echo out2_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache2" + out3_release_windows_i686_cache: + folder: out_cache3 + fingerprint_script: + - "echo out3_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache3" + git_release_windows_i686_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + gitindex_release_windows_i686_cache: + folder: git_index + fingerprint_script: + - "echo gitindex_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p git_index" + interrupted_aa_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_windows_i686" + reupload_on_changes: true + interrupted_ab_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_windows_i686" + reupload_on_changes: true + interrupted_ac_release_windows_i686_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_windows_i686" + reupload_on_changes: true + checkpoint_background_script: + # 110m caused the 2hr task timeout to be hit while the cache was uploading + # for macosx-toolchain, which broke subsequent builds. + - sleep 105m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh clang release windows i686 1" + env: + SIGN_BUILD: 0 + env: + CIRRUS_LOG_TIMESTAMP: true + BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 + depends_on: + - "release_windows_i686_download" + release_windows_i686_mingw-w64_para1_task: compute_engine_instance: image_project: cirrus-images @@ -2092,7 +2416,7 @@ release_windows_i686_mingw-w64_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_windows_i686_download" + - "release_windows_i686_clang_para1" release_windows_i686_goeasyconfig_1_task: compute_engine_instance: @@ -2660,6 +2984,87 @@ release_osx_x86_64_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 +release_osx_x86_64_clang_para1_task: + compute_engine_instance: + image_project: cirrus-images + image: family/docker-builder + platform: linux + cpu: 8 + memory: 16G + timeout_in: 120m + out_release_osx_x86_64_cache: + folder: out + fingerprint_script: + - "echo out_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" + out2_release_osx_x86_64_cache: + folder: out_cache2 + fingerprint_script: + - "echo out2_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache2" + out3_release_osx_x86_64_cache: + folder: out_cache3 + fingerprint_script: + - "echo out3_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache3" + git_release_osx_x86_64_cache: + folder: git_clones + fingerprint_script: + - "echo git_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_clones" + gitindex_release_osx_x86_64_cache: + folder: git_index + fingerprint_script: + - "echo gitindex_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p git_index" + interrupted_aa_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partaa.folder + fingerprint_script: + - "echo interrupted_aa_release_osx_x86_64" + reupload_on_changes: true + interrupted_ab_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partab.folder + fingerprint_script: + - "echo interrupted_ab_release_osx_x86_64" + reupload_on_changes: true + interrupted_ac_release_osx_x86_64_cache: + folder: tmp/interrupted_dirs.tar.gz.partac.folder + fingerprint_script: + - "echo interrupted_ac_release_osx_x86_64" + reupload_on_changes: true + checkpoint_background_script: + # 110m caused the 2hr task timeout to be hit while the cache was uploading + # for macosx-toolchain, which broke subsequent builds. + - sleep 105m + - ./tools/container-interrupt.sh + build_script: + - "./tools/cirrus_build_project.sh clang release osx x86_64 1" + env: + SIGN_BUILD: 0 + env: + CIRRUS_LOG_TIMESTAMP: true + BUMP_DEPS: 0 + RBM_NUM_PROCS: 8 + depends_on: + - "release_osx_x86_64_download" + release_osx_x86_64_macosx-toolchain_para1_task: compute_engine_instance: image_project: cirrus-images @@ -2739,7 +3144,7 @@ release_osx_x86_64_macosx-toolchain_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_osx_x86_64_download" + - "release_osx_x86_64_clang_para1" release_osx_x86_64_goeasyconfig_1_task: compute_engine_instance: diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index b42f5da..f4da863 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -83,7 +83,7 @@ print_os_arch () { # TODO fine-tune this list # Use "para" prefix to run with 8 threads; otherwise will use 1 thread. - for PROJECT in compiler.para1 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do + for PROJECT in clang.para1 compiler.para1 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) if [[ "$PROJECT_BASE" == "compiler" ]]; then if [[ "$OS" == "android" ]]; then @@ -203,7 +203,7 @@ print_os_arch () { fi # Depend on previous project - if [[ "$PROJECT" == "compiler.para1" ]]; then + if [[ "$PROJECT" == "clang.para1" ]]; then echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_download\"" else From 4da8ce078a85e9ea3235ba3b8dcb1ffdaea1e8fc Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Mon, 21 Mar 2022 19:08:37 +0000 Subject: [PATCH 4/4] Cirrus: Only build Clang on osx --- .cirrus.yml | 348 ++-------------------------------------- tools/cirrus_gen_yml.sh | 24 ++- 2 files changed, 28 insertions(+), 344 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 0426d56..6325e53 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,6 @@ # This file is auto-generated by tools/cirrus_gen_yml.sh -- do not edit manually! -release_linux_x86_64_download_task: +release_linux_x86_64_download_1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -72,87 +72,6 @@ release_linux_x86_64_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 -release_linux_x86_64_clang_para1_task: - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 8 - memory: 16G - timeout_in: 120m - out_release_linux_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_linux_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_linux_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_linux_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_linux_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_linux_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_linux_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_linux_x86_64" - reupload_on_changes: true - interrupted_ab_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_linux_x86_64" - reupload_on_changes: true - interrupted_ac_release_linux_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_linux_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh clang release linux x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - RBM_NUM_PROCS: 8 - depends_on: - - "release_linux_x86_64_download" - release_linux_x86_64_gcc_para1_task: compute_engine_instance: image_project: cirrus-images @@ -232,7 +151,7 @@ release_linux_x86_64_gcc_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_linux_x86_64_clang_para1" + - "release_linux_x86_64_download_1" release_linux_x86_64_goeasyconfig_1_task: compute_engine_instance: @@ -728,7 +647,7 @@ release_linux_x86_64_release_sign_task: depends_on: - "release_linux_x86_64_plain-binaries_1" -release_linux_i686_download_task: +release_linux_i686_download_1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -800,87 +719,6 @@ release_linux_i686_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 -release_linux_i686_clang_para1_task: - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 8 - memory: 16G - timeout_in: 120m - out_release_linux_i686_cache: - folder: out - fingerprint_script: - - "echo out_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_linux_i686_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_linux_i686_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_linux_i686_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_linux_i686_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_linux_i686_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_linux_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_linux_i686" - reupload_on_changes: true - interrupted_ab_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_linux_i686" - reupload_on_changes: true - interrupted_ac_release_linux_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_linux_i686" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh clang release linux i686 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - RBM_NUM_PROCS: 8 - depends_on: - - "release_linux_i686_download" - release_linux_i686_gcc_para1_task: compute_engine_instance: image_project: cirrus-images @@ -960,7 +798,7 @@ release_linux_i686_gcc_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_linux_i686_clang_para1" + - "release_linux_i686_download_1" release_linux_i686_goeasyconfig_1_task: compute_engine_instance: @@ -1456,7 +1294,7 @@ release_linux_i686_release_sign_task: depends_on: - "release_linux_i686_plain-binaries_1" -release_windows_x86_64_download_task: +release_windows_x86_64_download_1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -1528,87 +1366,6 @@ release_windows_x86_64_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 -release_windows_x86_64_clang_para1_task: - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 8 - memory: 16G - timeout_in: 120m - out_release_windows_x86_64_cache: - folder: out - fingerprint_script: - - "echo out_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_windows_x86_64_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_windows_x86_64_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_windows_x86_64_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_windows_x86_64_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_windows_x86_64_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_windows_x86_64" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_windows_x86_64" - reupload_on_changes: true - interrupted_ab_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_windows_x86_64" - reupload_on_changes: true - interrupted_ac_release_windows_x86_64_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_windows_x86_64" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh clang release windows x86_64 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - RBM_NUM_PROCS: 8 - depends_on: - - "release_windows_x86_64_download" - release_windows_x86_64_mingw-w64_para1_task: compute_engine_instance: image_project: cirrus-images @@ -1688,7 +1445,7 @@ release_windows_x86_64_mingw-w64_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_windows_x86_64_clang_para1" + - "release_windows_x86_64_download_1" release_windows_x86_64_goeasyconfig_1_task: compute_engine_instance: @@ -2184,7 +1941,7 @@ release_windows_x86_64_release_sign_task: depends_on: - "release_windows_x86_64_plain-binaries_1" -release_windows_i686_download_task: +release_windows_i686_download_1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -2256,87 +2013,6 @@ release_windows_i686_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 -release_windows_i686_clang_para1_task: - compute_engine_instance: - image_project: cirrus-images - image: family/docker-builder - platform: linux - cpu: 8 - memory: 16G - timeout_in: 120m - out_release_windows_i686_cache: - folder: out - fingerprint_script: - - "echo out_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out" - out1_release_windows_i686_cache: - folder: out_cache1 - fingerprint_script: - - "echo out1_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache1" - out2_release_windows_i686_cache: - folder: out_cache2 - fingerprint_script: - - "echo out2_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache2" - out3_release_windows_i686_cache: - folder: out_cache3 - fingerprint_script: - - "echo out3_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p out_cache3" - git_release_windows_i686_cache: - folder: git_clones - fingerprint_script: - - "echo git_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_clones" - gitindex_release_windows_i686_cache: - folder: git_index - fingerprint_script: - - "echo gitindex_release_windows_i686" - reupload_on_changes: true - populate_script: - - "mkdir -p git_index" - interrupted_aa_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partaa.folder - fingerprint_script: - - "echo interrupted_aa_release_windows_i686" - reupload_on_changes: true - interrupted_ab_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partab.folder - fingerprint_script: - - "echo interrupted_ab_release_windows_i686" - reupload_on_changes: true - interrupted_ac_release_windows_i686_cache: - folder: tmp/interrupted_dirs.tar.gz.partac.folder - fingerprint_script: - - "echo interrupted_ac_release_windows_i686" - reupload_on_changes: true - checkpoint_background_script: - # 110m caused the 2hr task timeout to be hit while the cache was uploading - # for macosx-toolchain, which broke subsequent builds. - - sleep 105m - - ./tools/container-interrupt.sh - build_script: - - "./tools/cirrus_build_project.sh clang release windows i686 1" - env: - SIGN_BUILD: 0 - env: - CIRRUS_LOG_TIMESTAMP: true - BUMP_DEPS: 0 - RBM_NUM_PROCS: 8 - depends_on: - - "release_windows_i686_download" - release_windows_i686_mingw-w64_para1_task: compute_engine_instance: image_project: cirrus-images @@ -2416,7 +2092,7 @@ release_windows_i686_mingw-w64_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_windows_i686_clang_para1" + - "release_windows_i686_download_1" release_windows_i686_goeasyconfig_1_task: compute_engine_instance: @@ -2912,7 +2588,7 @@ release_windows_i686_release_sign_task: depends_on: - "release_windows_i686_plain-binaries_1" -release_osx_x86_64_download_task: +release_osx_x86_64_download_1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -2984,7 +2660,7 @@ release_osx_x86_64_download_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 1 -release_osx_x86_64_clang_para1_task: +release_osx_x86_64_clang_osx-para1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -3063,7 +2739,7 @@ release_osx_x86_64_clang_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_osx_x86_64_download" + - "release_osx_x86_64_download_1" release_osx_x86_64_macosx-toolchain_para1_task: compute_engine_instance: @@ -3144,7 +2820,7 @@ release_osx_x86_64_macosx-toolchain_para1_task: BUMP_DEPS: 0 RBM_NUM_PROCS: 8 depends_on: - - "release_osx_x86_64_clang_para1" + - "release_osx_x86_64_clang_osx-para1" release_osx_x86_64_goeasyconfig_1_task: compute_engine_instance: diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index f4da863..2cd8943 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -8,7 +8,7 @@ print_os_arch () { local ARCH="$2" # Pre-download tarballs and Git repos - echo "${CHANNEL}_${OS}_${ARCH}_download_task: + echo "${CHANNEL}_${OS}_${ARCH}_download_1_task: compute_engine_instance: image_project: cirrus-images image: family/docker-builder @@ -81,9 +81,13 @@ print_os_arch () { RBM_NUM_PROCS: 1" echo "" + local PREV_PROJECT_BASE="download" + local PREV_PROJECT_ITER="1" + # TODO fine-tune this list # Use "para" prefix to run with 8 threads; otherwise will use 1 thread. - for PROJECT in clang.para1 compiler.para1 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do + # Use "osx" prefix to run exclusively for that OS; otherwise will run for all OS's. + for PROJECT in clang.osx-para1 compiler.para1 goeasyconfig.1 ncdns.1 ncprop279.1 plain-binaries.1 release.nosign release.sign; do PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1) if [[ "$PROJECT_BASE" == "compiler" ]]; then if [[ "$OS" == "android" ]]; then @@ -99,13 +103,22 @@ print_os_arch () { PROJECT_BASE=macosx-toolchain fi fi + PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2) + + # Skip inapplicable OS-specific projects + if echo $PROJECT_ITER | grep -q osx && [[ "$OS" != "osx" ]] ; then + continue + fi + + # Run heavy projects with maximum thread count PARA_THREADS=1 PARA_RAM=3 if echo $PROJECT_ITER | grep -q para ; then PARA_THREADS=8 PARA_RAM=16 fi + echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_task: compute_engine_instance: image_project: cirrus-images @@ -203,13 +216,8 @@ print_os_arch () { fi # Depend on previous project - if [[ "$PROJECT" == "clang.para1" ]]; then - echo " depends_on: - - \"${CHANNEL}_${OS}_${ARCH}_download\"" - else - echo " depends_on: + echo " depends_on: - \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT_BASE}_${PREV_PROJECT_ITER}\"" - fi if [[ "$PROJECT_ITER" != "nosign" ]]; then local PREV_PROJECT_BASE="$PROJECT_BASE"