From 25ccedfb1b80f0f1db73b072ede03a2bcbf0c0cc Mon Sep 17 00:00:00 2001 From: Jeremy Rand Date: Sat, 20 Feb 2021 20:13:17 +0000 Subject: [PATCH] Cirrus: Split off macosx-toolchain output cache --- .cirrus.yml | 315 ++++++++++++++++++++++++++++++++++ tools/cirrus_build_project.sh | 23 +-- tools/cirrus_gen_yml.sh | 14 ++ 3 files changed, 335 insertions(+), 17 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index c3f2945..f676336 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -7,6 +7,13 @@ release_linux_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -41,6 +48,13 @@ release_linux_x86_64_gcc_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -80,6 +94,13 @@ release_linux_x86_64_gcc_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -119,6 +140,13 @@ release_linux_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -158,6 +186,13 @@ release_linux_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -197,6 +232,13 @@ release_linux_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -236,6 +278,13 @@ release_linux_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -275,6 +324,13 @@ release_linux_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -314,6 +370,13 @@ release_linux_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_x86_64_cache: folder: git_clones fingerprint_script: @@ -353,6 +416,13 @@ release_linux_i686_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -387,6 +457,13 @@ release_linux_i686_gcc_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -426,6 +503,13 @@ release_linux_i686_gcc_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -465,6 +549,13 @@ release_linux_i686_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -504,6 +595,13 @@ release_linux_i686_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -543,6 +641,13 @@ release_linux_i686_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -582,6 +687,13 @@ release_linux_i686_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -621,6 +733,13 @@ release_linux_i686_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -660,6 +779,13 @@ release_linux_i686_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_linux_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_linux_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_linux_i686_cache: folder: git_clones fingerprint_script: @@ -699,6 +825,13 @@ release_windows_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -733,6 +866,13 @@ release_windows_x86_64_mingw-w64_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -772,6 +912,13 @@ release_windows_x86_64_mingw-w64_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -811,6 +958,13 @@ release_windows_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -850,6 +1004,13 @@ release_windows_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -889,6 +1050,13 @@ release_windows_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -928,6 +1096,13 @@ release_windows_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -967,6 +1142,13 @@ release_windows_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -1006,6 +1188,13 @@ release_windows_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_x86_64_cache: folder: git_clones fingerprint_script: @@ -1045,6 +1234,13 @@ release_windows_i686_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1079,6 +1275,13 @@ release_windows_i686_mingw-w64_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1118,6 +1321,13 @@ release_windows_i686_mingw-w64_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1157,6 +1367,13 @@ release_windows_i686_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1196,6 +1413,13 @@ release_windows_i686_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1235,6 +1459,13 @@ release_windows_i686_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1274,6 +1505,13 @@ release_windows_i686_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1313,6 +1551,13 @@ release_windows_i686_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1352,6 +1597,13 @@ release_windows_i686_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_windows_i686_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_windows_i686" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_windows_i686_cache: folder: git_clones fingerprint_script: @@ -1391,6 +1643,13 @@ release_osx_x86_64_download_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1425,6 +1684,13 @@ release_osx_x86_64_macosx-toolchain_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1464,6 +1730,13 @@ release_osx_x86_64_macosx-toolchain_2_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1503,6 +1776,13 @@ release_osx_x86_64_goeasyconfig_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1542,6 +1822,13 @@ release_osx_x86_64_ncdns_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1581,6 +1868,13 @@ release_osx_x86_64_ncp11_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1620,6 +1914,13 @@ release_osx_x86_64_ncprop279_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1659,6 +1960,13 @@ release_osx_x86_64_plain-binaries_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: @@ -1698,6 +2006,13 @@ release_osx_x86_64_release_1_docker_builder: reupload_on_changes: true populate_script: - "mkdir -p out" + out1_release_osx_x86_64_cache: + folder: out_cache1 + fingerprint_script: + - "echo out1_release_osx_x86_64" + reupload_on_changes: true + populate_script: + - "mkdir -p out_cache1" git_release_osx_x86_64_cache: folder: git_clones fingerprint_script: diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh index 27acdc4..fb10ae0 100755 --- a/tools/cirrus_build_project.sh +++ b/tools/cirrus_build_project.sh @@ -32,14 +32,8 @@ patch -p1 < ../tools/checkpoints.patch popd echo "Moving caches..." -if [[ -e "./fonts/.git" ]]; then - echo "git_clones/fonts was cached, moving it to the right place..." - mv ./fonts ./git_clones/fonts -else - echo "git_clones/fonts was not cached." - rm -rf ./fonts ./git_clones/fonts -fi -ls ./git_clones +mv ./out_cache1/* ./out/ || true +rm -rf ./out_cache1/* || true echo "Unpacking interrupted cache..." ./tools/cirrus_unpack_interrupted.sh || true @@ -77,15 +71,6 @@ else rm -rf ./tmp/interrupted_dirs/* || true fi -echo "Moving caches..." -if [[ -e "git_clones/fonts" ]]; then - echo "git_clones/fonts is ready to be cached, moving it to the right place..." - mv git_clones/fonts ./ -else - echo "git_clones/fonts is not ready to be not cached." - mkdir -p ./fonts -fi - # The cache has a size limit, so we need to clean useless data from it. The # container-images are very large and seem to be fairly harmless to remove. # Maybe later if we have more pressure to shrink, we could remove the @@ -93,5 +78,9 @@ fi echo "Cleaning cache..." rm -rfv out/container-image +echo "Moving caches..." +mv ./out/macosx-toolchain ./out_cache1/ || true +rm -rf ./out/macosx-toolchain || true + echo "Packing interrupted cache..." ./tools/cirrus_pack_interrupted.sh || true diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh index 56fbb15..c7822e9 100755 --- a/tools/cirrus_gen_yml.sh +++ b/tools/cirrus_gen_yml.sh @@ -17,6 +17,13 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p out\" + out1_${CHANNEL}_${OS}_${ARCH}_cache: + folder: out_cache1 + fingerprint_script: + - \"echo out1_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p out_cache1\" git_${CHANNEL}_${OS}_${ARCH}_cache: folder: git_clones fingerprint_script: @@ -70,6 +77,13 @@ print_os_arch () { reupload_on_changes: true populate_script: - \"mkdir -p out\" + out1_${CHANNEL}_${OS}_${ARCH}_cache: + folder: out_cache1 + fingerprint_script: + - \"echo out1_${CHANNEL}_${OS}_${ARCH}\" + reupload_on_changes: true + populate_script: + - \"mkdir -p out_cache1\" git_${CHANNEL}_${OS}_${ARCH}_cache: folder: git_clones fingerprint_script: