Merge #88: Add Cirrus rbm builds
pull/91/head2a98623Cirrus: Allow rsync to fail (Jeremy Rand)050d0e4Cirrus: Run "make clean" dry-run (Jeremy Rand)1dda171Cirrus: Make output cache a bit more robust (Jeremy Rand)25ccedfCirrus: Split off macosx-toolchain output cache (Jeremy Rand)aa27f98Cirrus: Factor out compiler project into its own tasks (Jeremy Rand)bb777f9Cirrus: Expose checkpoint cache to download task (Jeremy Rand)a5596fdCirrus: clear checkpoint on download task (Jeremy Rand)2a83d0aPatch macosx-toolchain for checkpoints (Jeremy Rand)e6713ecCirrus: add ncp11 and ncprop279 to download task (Jeremy Rand)ef3d6a1Make checkpoint cache upload/download more robust (Jeremy Rand)e18caa0Fix interrupted cache dir for projects without a version (Jeremy Rand)05fb0e2Cirrus: fix empty checkpoint directory detection (Jeremy Rand)f0a34d6Cirrus: Enable builds for macOS (Jeremy Rand)3f89128Cirrus: Add 2nd iteration for goeasyconfig (Jeremy Rand)e0b4802Cirrus: cache checkpoints (Jeremy Rand)2c78220Cirrus: Don't fail a task if it saved a checkpoint (Jeremy Rand)07b6abbCirrus: Add checkpoint background script (Jeremy Rand)1e533e5Add echos to interrupt script (Jeremy Rand)0796ce6Cirrus: Apply checkpoint patch to tor-browser-build (Jeremy Rand)ea1b2edAdd tor-browser-build checkpoint patch (Jeremy Rand)0c1c248rbm.conf: Handle SIGINT case (Jeremy Rand)a72f41bAdd container-interrupt script (Jeremy Rand)180eeb4Add container-pids tool (Jeremy Rand)ad526b5Fix build script for release project (Jeremy Rand)0113996Cirrus: Add download task for macOS (Jeremy Rand)9175e0dCirrus: Print logs from rbm containers (Jeremy Rand)1c02460Add rbm.local.conf.example (Jeremy Rand)4243f16Cirrus: Don't try to cache release project (Jeremy Rand)eab6460Cirrus: Try apt-get twice (Jeremy Rand)4140483Add libtapi project (Jeremy Rand)7c8ab70Add release project (Jeremy Rand)e296c02Add clang-source and ninja (Jeremy Rand)8a14198Add plain-binaries project (Jeremy Rand)3244cf7Cirrus: Add Windows targets (Jeremy Rand)05cc382Cirrus: Add linux-i686 target (Jeremy Rand)32b4c55Bump tor-browser-build (Jeremy Rand)4a4a50dAdd Cirrus task for goeasyconfig (Jeremy Rand)d364bf8Bump ncdns version (Jeremy Rand)cd60babBump tor-browser-build (Jeremy Rand)4aaf6bfBump rbm.conf (Jeremy Rand)430dda3Bump tor-browser-build (Jeremy Rand)d62b523Cirrus: Namecoin doesn't have a Nightly channel (Jeremy Rand)79c575aAdd skeleton of Cirrus rbm builds (Jeremy Rand) Pull request description: This PR enables building our rbm projects on Cirrus CI. This code is intended to be submitted upstream to Tor Browser in the future, but for now Namecoin will incubate it until I'm convinced it's stable.
Jeremy Rand
3 years ago
commit
10a4948fec
File diff suppressed because it is too large
Load Diff
@ -0,0 +1 @@
|
||||
../tor-browser-build/projects/clang-source
|
||||
@ -0,0 +1,16 @@
|
||||
version: '[% c("abbrev") %]'
|
||||
git_url: https://github.com/namecoin/splicesign.git
|
||||
git_hash: 38bb6fb3ec66c72ecb3a14e1e714768cc6e56ed7
|
||||
filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
|
||||
|
||||
build: '[% c("projects/go/var/build_go_lib") %]'
|
||||
|
||||
var:
|
||||
container:
|
||||
use_container: 1
|
||||
go_lib: github.com/namecoin/splicesign
|
||||
|
||||
input_files:
|
||||
- project: container-image
|
||||
- name: go
|
||||
project: go
|
||||
@ -0,0 +1 @@
|
||||
../tor-browser-build/projects/libtapi
|
||||
@ -0,0 +1 @@
|
||||
../tor-browser-build/projects/ninja
|
||||
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
[% c("var/set_default_env") -%]
|
||||
|
||||
distdir=/var/tmp/dist/[% project %]
|
||||
mkdir -p $distdir
|
||||
|
||||
tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncdns') %]
|
||||
tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncp11') %]
|
||||
tar -C /var/tmp/dist/[% project %] -xf [% c('input_files_by_name/ncprop279') %]
|
||||
|
||||
cd $distdir
|
||||
[% c('tar', {
|
||||
tar_src => [ '.' ],
|
||||
tar_args => '-czf ' _ dest_dir _ '/' _ c('filename'),
|
||||
}) %]
|
||||
@ -0,0 +1,15 @@
|
||||
version: '[% c("var/ncdns_version") %]'
|
||||
filename: '[% project %]-[% c("version") %]-[% c("var/osname") %]-[% c("var/build_id") %].tar.gz'
|
||||
|
||||
var:
|
||||
container:
|
||||
use_container: 1
|
||||
|
||||
input_files:
|
||||
- project: container-image
|
||||
- name: ncdns
|
||||
project: ncdns
|
||||
- name: ncp11
|
||||
project: ncp11
|
||||
- name: ncprop279
|
||||
project: ncprop279
|
||||
@ -0,0 +1,46 @@
|
||||
#!/bin/sh
|
||||
[% c("var/set_default_env") -%]
|
||||
# reset HOME which was changed by var/set_default_env, for gpg
|
||||
[% IF ENV.HOME %]export HOME="[% ENV.HOME %]"[% END %]
|
||||
destdir="[% dest_dir _ '/' _ c("var/publish_dir") %]"
|
||||
mkdir -p "$destdir"
|
||||
[% IF c("var/ncdns-android-armv7") -%]
|
||||
mv [% c('input_files_by_name/android-armv7') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-android-x86") -%]
|
||||
mv [% c('input_files_by_name/android-x86') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-android-x86_64") -%]
|
||||
mv [% c('input_files_by_name/android-x86_64') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-android-aarch64") -%]
|
||||
mv [% c('input_files_by_name/android-aarch64') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-windows-i686") -%]
|
||||
mv [% c('input_files_by_name/windows-i686') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-windows-x86_64") -%]
|
||||
mv [% c('input_files_by_name/windows-x86_64') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-osx-x86_64") -%]
|
||||
mv [% c('input_files_by_name/osx-x86_64') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-linux-i686") -%]
|
||||
mv [% c('input_files_by_name/linux-i686') %] "$destdir"/
|
||||
[% END -%]
|
||||
[% IF c("var/ncdns-linux-x86_64") -%]
|
||||
mv [% c('input_files_by_name/linux-x86_64') %] "$destdir"/
|
||||
[% END -%]
|
||||
cd "$destdir"
|
||||
cat > .htaccess <<EOF
|
||||
RewriteEngine On
|
||||
RewriteRule ^sha256sums.txt$ sha256sums-unsigned-build.txt
|
||||
RewriteRule ^sha256sums.txt.asc$ sha256sums-unsigned-build.txt.asc
|
||||
RewriteRule ^sha256sums.incrementals.txt$ sha256sums-unsigned-build.incrementals.txt
|
||||
RewriteRule ^sha256sums.incrementals.txt.asc$ sha256sums-unsigned-build.incrementals.txt.asc
|
||||
EOF
|
||||
sha256sum $(ls -1 *.exe *.tar.xz *.dmg *.mar *.zip *.tar.gz *.apk *.json | grep -v '\.incremental\.mar$' | sort) > sha256sums-unsigned-build.txt
|
||||
[% IF c("var/sign_build") -%]
|
||||
gpg -abs [% c("var/sign_build_gpg_opts") %] sha256sums-unsigned-build.txt
|
||||
[% END -%]
|
||||
cat sha256sums-unsigned-build.txt
|
||||
@ -0,0 +1,184 @@
|
||||
# vim: filetype=yaml sw=2
|
||||
version: '[% c("var/ncdns_version") %]'
|
||||
output_dir: release
|
||||
|
||||
var:
|
||||
signed_status: unsigned
|
||||
publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/ncdns_build") %]'
|
||||
containers_target: with_containers
|
||||
|
||||
targets:
|
||||
ncdns-all:
|
||||
- ncdns-linux-x86_64
|
||||
- ncdns-linux-i686
|
||||
- ncdns-windows-i686
|
||||
- ncdns-windows-x86_64
|
||||
- ncdns-osx-x86_64
|
||||
#- ncdns-android-armv7
|
||||
#- ncdns-android-x86
|
||||
#- ncdns-android-x86_64
|
||||
#- ncdns-android-aarch64
|
||||
- ncdns-src
|
||||
ncdns-all-desktop:
|
||||
- ncdns-linux-x86_64
|
||||
- ncdns-linux-i686
|
||||
- ncdns-windows-i686
|
||||
- ncdns-windows-x86_64
|
||||
- ncdns-osx-x86_64
|
||||
- ncdns-src
|
||||
ncdns-all-android:
|
||||
- ncdns-android-armv7
|
||||
- ncdns-android-x86
|
||||
- ncdns-android-x86_64
|
||||
- ncdns-android-aarch64
|
||||
ncdns-android-armv7:
|
||||
var:
|
||||
ncdns-android-armv7: 1
|
||||
ncdns-android-x86:
|
||||
var:
|
||||
ncdns-android-x86: 1
|
||||
ncdns-android-x86_64:
|
||||
var:
|
||||
ncdns-android-x86_64: 1
|
||||
ncdns-android-aarch64:
|
||||
var:
|
||||
ncdns-android-aarch64: 1
|
||||
ncdns-linux-x86_64:
|
||||
var:
|
||||
ncdns-linux-x86_64: 1
|
||||
ncdns-linux-x86_64-asan:
|
||||
var:
|
||||
ncdns-linux-x86_64: 1
|
||||
asan-build: '-asan'
|
||||
ncdns-linux-i686:
|
||||
var:
|
||||
ncdns-linux-i686: 1
|
||||
ncdns-windows-i686:
|
||||
var:
|
||||
ncdns-windows-i686: 1
|
||||
ncdns-windows-x86_64:
|
||||
var:
|
||||
ncdns-windows-x86_64: 1
|
||||
ncdns-osx-x86_64:
|
||||
var:
|
||||
ncdns-osx-x86_64: 1
|
||||
ncdns-src:
|
||||
var:
|
||||
ncdns-src: '[% ! c("var/testbuild") %]'
|
||||
ncdns-src-testbuild:
|
||||
var:
|
||||
ncdns-src: 1
|
||||
|
||||
notarget:
|
||||
- release
|
||||
- ncdns-all
|
||||
#- ncdns-all-android
|
||||
- ncdns-all-desktop
|
||||
|
||||
noversiondir:
|
||||
var:
|
||||
publish_dir: ''
|
||||
|
||||
release:
|
||||
var:
|
||||
build_target: release
|
||||
|
||||
nightly:
|
||||
output_dir: 'nightly'
|
||||
var:
|
||||
build_target: nightly
|
||||
publish_dir: '[% c("version") %]'
|
||||
|
||||
alpha:
|
||||
output_dir: alpha
|
||||
var:
|
||||
build_target: alpha
|
||||
|
||||
testbuild:
|
||||
output_dir: 'testbuild'
|
||||
var:
|
||||
testbuild: 1
|
||||
build_target: ncdns-testbuild
|
||||
publish_dir: ''
|
||||
|
||||
no_containers:
|
||||
var:
|
||||
containers_target: no_containers
|
||||
|
||||
signed:
|
||||
var:
|
||||
signed_status: signed
|
||||
|
||||
create_unsigned_incrementals:
|
||||
var:
|
||||
create_unsigned_incrementals: 1
|
||||
|
||||
input_files:
|
||||
|
||||
# Release
|
||||
- name: android-armv7
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-android-armv7") %]'
|
||||
target:
|
||||
- '[% c("var/containers_target") %]'
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-android-armv7
|
||||
|
||||
- name: android-x86
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-android-x86") %]'
|
||||
target:
|
||||
- '[% c("var/containers_target") %]'
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-android-x86
|
||||
|
||||
- name: android-x86_64
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-android-x86_64") %]'
|
||||
target:
|
||||
- '[% c("var/containers_target") %]'
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-android-x86_64
|
||||
|
||||
- name: android-aarch64
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-android-aarch64") %]'
|
||||
target:
|
||||
- '[% c("var/containers_target") %]'
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-android-aarch64
|
||||
|
||||
- name: linux-x86_64
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-linux-x86_64") %]'
|
||||
target:
|
||||
- '[% c("var/build_target") %]'
|
||||
- 'ncdns-linux-x86_64[% c("var/asan-build") %]'
|
||||
|
||||
- name: linux-i686
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-linux-i686") %]'
|
||||
target:
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-linux-i686
|
||||
|
||||
- name: windows-i686
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-windows-i686") %]'
|
||||
target:
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-windows-i686
|
||||
|
||||
- name: windows-x86_64
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-windows-x86_64") %]'
|
||||
target:
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-windows-x86_64
|
||||
|
||||
- name: osx-x86_64
|
||||
project: plain-binaries
|
||||
enable: '[% c("var/ncdns-osx-x86_64") %]'
|
||||
target:
|
||||
- '[% c("var/build_target") %]'
|
||||
- ncdns-osx-x86_64
|
||||
@ -0,0 +1 @@
|
||||
tor-browser-build/rbm.local.conf.example
|
||||
@ -0,0 +1,86 @@
|
||||
From 25b581c833679939408fa51ce1d43b55ae481d51 Mon Sep 17 00:00:00 2001
|
||||
From: Jeremy Rand <jeremyrand@airmail.cc>
|
||||
Date: Fri, 19 Feb 2021 00:23:40 +0000
|
||||
Subject: [PATCH 1/2] clang: Support rbm checkpointing
|
||||
|
||||
---
|
||||
projects/clang/build | 13 +++++++++++++
|
||||
1 file changed, 13 insertions(+)
|
||||
|
||||
diff --git a/projects/clang/build b/projects/clang/build
|
||||
index c4c16d9..bb8283b 100644
|
||||
--- a/projects/clang/build
|
||||
+++ b/projects/clang/build
|
||||
@@ -2,6 +2,13 @@
|
||||
[% c("var/set_default_env") -%]
|
||||
distdir=/var/tmp/dist/[% project %]
|
||||
mkdir -p /var/tmp/dist
|
||||
+
|
||||
+if [[ -e /var/tmp/dist/checkpoint1 ]] ; then
|
||||
+ set +e
|
||||
+ source /var/tmp/dist/checkpoint1
|
||||
+ set -e
|
||||
+else
|
||||
+
|
||||
tar -C /var/tmp/dist -xf [% c('input_files_by_name/cmake') %]
|
||||
export PATH="/var/tmp/dist/cmake/bin:$PATH"
|
||||
[% IF c("var/linux") || c("var/android") %]
|
||||
@@ -35,6 +42,12 @@ cd build
|
||||
[% IF c("var/rlbox") -%]-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \[% END -%]
|
||||
-DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;lld"
|
||||
[% END -%]
|
||||
+
|
||||
+set > /var/tmp/dist/checkpoint1
|
||||
+export -p >> /var/tmp/dist/checkpoint1
|
||||
+fi
|
||||
+
|
||||
+cd /var/tmp/build/clang-source/build
|
||||
make -j[% c("buildconf/num_procs") %]
|
||||
make install
|
||||
cd /var/tmp/dist
|
||||
--
|
||||
2.20.1
|
||||
|
||||
|
||||
From ab0c171fe1c82efed5cbaf657b06d7103f209cb9 Mon Sep 17 00:00:00 2001
|
||||
From: Jeremy Rand <jeremyrand@airmail.cc>
|
||||
Date: Sat, 20 Feb 2021 08:51:15 +0000
|
||||
Subject: [PATCH 2/2] macosx-toolchain: Support rbm checkpointing
|
||||
|
||||
---
|
||||
projects/macosx-toolchain/build | 13 +++++++++++++
|
||||
1 file changed, 13 insertions(+)
|
||||
|
||||
diff --git a/projects/macosx-toolchain/build b/projects/macosx-toolchain/build
|
||||
index 5716a9e..94f6df1 100644
|
||||
--- a/projects/macosx-toolchain/build
|
||||
+++ b/projects/macosx-toolchain/build
|
||||
@@ -1,5 +1,12 @@
|
||||
#!/bin/bash
|
||||
[% c("var/set_default_env") -%]
|
||||
+
|
||||
+if [[ -e /var/tmp/dist/checkpoint1 ]] ; then
|
||||
+ set +e
|
||||
+ source /var/tmp/dist/checkpoint1
|
||||
+ set -e
|
||||
+else
|
||||
+
|
||||
builddir=/var/tmp/build
|
||||
mkdir $builddir
|
||||
distdir=/var/tmp/dist/[% project %]
|
||||
@@ -65,6 +72,12 @@ cmake -GNinja \
|
||||
-DDARWIN_osx_ARCHS=x86_64 \
|
||||
-DDARWIN_osx_SYSROOT=$sysrootdir \
|
||||
-DLLVM_DEFAULT_TARGET_TRIPLE=x86_64-apple-darwin $builddir/clang-source
|
||||
+
|
||||
+set > /var/tmp/dist/checkpoint1
|
||||
+export -p >> /var/tmp/dist/checkpoint1
|
||||
+fi
|
||||
+
|
||||
+cd $builddir/clang-source/build
|
||||
ninja install -v
|
||||
|
||||
# We now have a native macosx64 toolchain.
|
||||
--
|
||||
2.20.1
|
||||
|
||||
@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
shopt -s nullglob globstar
|
||||
|
||||
PROJECT=$1
|
||||
CHANNEL=$2
|
||||
OS=$3
|
||||
ARCH=$4
|
||||
SHOULD_BUILD=$5
|
||||
|
||||
echo "Checking VM specs..."
|
||||
cat /etc/*-release
|
||||
df -h
|
||||
lscpu
|
||||
free -m
|
||||
|
||||
echo "Installing rbm deps..."
|
||||
APT_DEPS="libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc rsync"
|
||||
apt-get install -y $APT_DEPS || (sleep 15s && apt-get install -y $APT_DEPS)
|
||||
|
||||
echo "Pulling rbm..."
|
||||
make submodule-update
|
||||
|
||||
echo "Configuring rbm..."
|
||||
# Print logs to Cirrus.
|
||||
cat rbm.local.conf.example | sed "s/#build_log: '-'/build_log: '-'/g" > rbm.local.conf
|
||||
# Configure "make clean"
|
||||
cat tools/rbm.local.conf.onetarget | sed "s/CHANNEL/$CHANNEL/g" | sed "s/ncdns-all/ncdns-$OS-$ARCH/g" >> rbm.local.conf
|
||||
|
||||
echo "Patching rbm..."
|
||||
pushd tor-browser-build
|
||||
patch -p1 < ../tools/checkpoints.patch
|
||||
popd
|
||||
|
||||
echo "Restoring caches..."
|
||||
cp -a ./out_cache1/* ./out/ || true
|
||||
|
||||
echo "Unpacking interrupted cache..."
|
||||
./tools/cirrus_unpack_interrupted.sh || true
|
||||
|
||||
if [[ "$PROJECT" == "release" ]]; then
|
||||
echo "release project is never cached."
|
||||
else
|
||||
echo "Checking if project is cached..."
|
||||
OUTDIR="$(./rbm/rbm showconf $PROJECT output_dir --target $CHANNEL --target ncdns-$OS-$ARCH)"
|
||||
OUTFILE="$(./rbm/rbm showconf $PROJECT filename --target $CHANNEL --target ncdns-$OS-$ARCH)"
|
||||
if [[ -e "$OUTDIR/$OUTFILE" ]]; then
|
||||
echo "Project cache hit, skipping build."
|
||||
SHOULD_BUILD=0
|
||||
else
|
||||
echo "Project cache miss, proceeding with build."
|
||||
fi
|
||||
fi
|
||||
|
||||
# VM has 12 GB of free RAM. Assuming each of the 4 logical cores takes 1 GB
|
||||
# during build, that leaves us with 8 GB of unutilized RAM. Alas, I'm not sure
|
||||
# that's enough, so this isn't enabled right now.
|
||||
#echo "Mounting tmpfs..."
|
||||
#mount -t tmpfs -o size=8G,nr_inodes=40k,mode=1777 tmpfs ./tmp
|
||||
#df -h
|
||||
|
||||
if [[ "$SHOULD_BUILD" -eq 1 ]]; then
|
||||
echo "Building project..."
|
||||
# If rbm fails, we consider it a success as long as it saved a checkpoint.
|
||||
./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH" || [ ! -z "$(ls -A ./tmp/interrupted_dirs/)" ]
|
||||
else
|
||||
#echo "This is a cache-only task, skipping build."
|
||||
echo "Skipping build."
|
||||
|
||||
echo "Clearing interrupted cache..."
|
||||
rm -rf ./tmp/interrupted_dirs/* || true
|
||||
fi
|
||||
|
||||
# The cache has a size limit, so we need to clean useless data from it. The
|
||||
# container-images are very large and seem to be fairly harmless to remove.
|
||||
# Maybe later if we have more pressure to shrink, we could remove the
|
||||
# debootstrap-images too.
|
||||
echo "Cleaning cache..."
|
||||
rm -rfv out/container-image
|
||||
if [[ "$SHOULD_BUILD" -eq 0 ]]; then
|
||||
./tools/clean-old --dry-run
|
||||
fi
|
||||
|
||||
echo "Splitting caches..."
|
||||
rsync -avu --delete ./out/macosx-toolchain ./out_cache1/ || true
|
||||
rm -rf ./out/macosx-toolchain || true
|
||||
|
||||
echo "Packing interrupted cache..."
|
||||
./tools/cirrus_pack_interrupted.sh || true
|
||||
@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
shopt -s nullglob globstar
|
||||
|
||||
print_os_arch () {
|
||||
local OS="$1"
|
||||
local ARCH="$2"
|
||||
|
||||
# Pre-download tarballs and Git repos
|
||||
echo "${CHANNEL}_${OS}_${ARCH}_download_docker_builder:
|
||||
timeout_in: 120m
|
||||
out_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: out
|
||||
fingerprint_script:
|
||||
- \"echo out_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
populate_script:
|
||||
- \"mkdir -p out\"
|
||||
out1_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: out_cache1
|
||||
fingerprint_script:
|
||||
- \"echo out1_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
populate_script:
|
||||
- \"mkdir -p out_cache1\"
|
||||
git_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: git_clones
|
||||
fingerprint_script:
|
||||
- \"echo git_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
populate_script:
|
||||
- \"mkdir -p git_clones\"
|
||||
interrupted_aa_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: tmp/interrupted_dirs.tar.gz.partaa.folder
|
||||
fingerprint_script:
|
||||
- \"echo interrupted_aa_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
interrupted_ab_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: tmp/interrupted_dirs.tar.gz.partab.folder
|
||||
fingerprint_script:
|
||||
- \"echo interrupted_ab_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
interrupted_ac_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: tmp/interrupted_dirs.tar.gz.partac.folder
|
||||
fingerprint_script:
|
||||
- \"echo interrupted_ac_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
build_script:
|
||||
- \"./tools/cirrus_build_project.sh plain-binaries ${CHANNEL} ${OS} ${ARCH} 0\""
|
||||
echo ""
|
||||
|
||||
# TODO fine-tune this list
|
||||
for PROJECT in compiler.1 compiler.2 goeasyconfig.1 ncdns.1 ncp11.1 ncprop279.1 plain-binaries.1 release.1; do
|
||||
PROJECT_BASE=$(echo $PROJECT | cut -d . -f 1)
|
||||
if [[ "$PROJECT_BASE" == "compiler" ]]; then
|
||||
if [[ "$OS" == "android" ]]; then
|
||||
PROJECT_BASE=android-toolchain
|
||||
fi
|
||||
if [[ "$OS" == "linux" ]]; then
|
||||
PROJECT_BASE=gcc
|
||||
fi
|
||||
if [[ "$OS" == "windows" ]]; then
|
||||
PROJECT_BASE=mingw-w64
|
||||
fi
|
||||
if [[ "$OS" == "osx" ]]; then
|
||||
PROJECT_BASE=macosx-toolchain
|
||||
fi
|
||||
fi
|
||||
PROJECT_ITER=$(echo $PROJECT | cut -d . -f 2)
|
||||
echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT_BASE}_${PROJECT_ITER}_docker_builder:
|
||||
timeout_in: 120m
|
||||
out_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: out
|
||||
fingerprint_script:
|
||||
- \"echo out_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
populate_script:
|
||||
- \"mkdir -p out\"
|
||||
out1_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: out_cache1
|
||||
fingerprint_script:
|
||||
- \"echo out1_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
populate_script:
|
||||
- \"mkdir -p out_cache1\"
|
||||
git_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: git_clones
|
||||
fingerprint_script:
|
||||
- \"echo git_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
populate_script:
|
||||
- \"mkdir -p git_clones\"
|
||||
interrupted_aa_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: tmp/interrupted_dirs.tar.gz.partaa.folder
|
||||
fingerprint_script:
|
||||
- \"echo interrupted_aa_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
interrupted_ab_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: tmp/interrupted_dirs.tar.gz.partab.folder
|
||||
fingerprint_script:
|
||||
- \"echo interrupted_ab_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
interrupted_ac_${CHANNEL}_${OS}_${ARCH}_cache:
|
||||
folder: tmp/interrupted_dirs.tar.gz.partac.folder
|
||||
fingerprint_script:
|
||||
- \"echo interrupted_ac_${CHANNEL}_${OS}_${ARCH}\"
|
||||
reupload_on_changes: true
|
||||
checkpoint_background_script:
|
||||
- sleep 110m
|
||||
- ./tools/container-interrupt.sh
|
||||
build_script:
|
||||
- \"./tools/cirrus_build_project.sh ${PROJECT_BASE} ${CHANNEL} ${OS} ${ARCH} 1\""
|
||||
|
||||
# Depend on previous project
|
||||
if [[ "$PROJECT" == "compiler.1" ]]; then
|
||||
echo " depends_on:
|
||||
- \"${CHANNEL}_${OS}_${ARCH}_download\""
|
||||
else
|
||||
echo " depends_on:
|
||||
- \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT_BASE}_${PREV_PROJECT_ITER}\""
|
||||
fi
|
||||
|
||||
local PREV_PROJECT_BASE="$PROJECT_BASE"
|
||||
local PREV_PROJECT_ITER="$PROJECT_ITER"
|
||||
echo ""
|
||||
done
|
||||
}
|
||||
|
||||
(
|
||||
for CHANNEL in release; do
|
||||
print_os_arch linux x86_64
|
||||
print_os_arch linux i686
|
||||
print_os_arch windows x86_64
|
||||
print_os_arch windows i686
|
||||
print_os_arch osx x86_64
|
||||
done
|
||||
) > .cirrus.yml
|
||||
|
||||
# Timeout issues?
|
||||
# Might want to increase the timeout -- but we're already using the 2 hour max.
|
||||
# Might want to bump the CPU count -- but that's blocked by cirrus-ci-docs issue #741.
|
||||
# Might want to split into smaller project sets.
|
||||
# What is the CPU count limit? "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 16.0 CPU. docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS).
|
||||
@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
shopt -s failglob
|
||||
|
||||
pushd tmp
|
||||
|
||||
mkdir -p interrupted_dirs
|
||||
tar -caf "interrupted_dirs.tar.gz" interrupted_dirs
|
||||
|
||||
CHUNKS=3
|
||||
|
||||
split --number=$CHUNKS interrupted_dirs.tar.gz interrupted_dirs.tar.gz.part
|
||||
|
||||
rm interrupted_dirs.tar.gz
|
||||
|
||||
for PART in interrupted_dirs.tar.gz.part* ; do
|
||||
mkdir -p $PART.folder
|
||||
mv $PART $PART.folder/
|
||||
done
|
||||
|
||||
popd
|
||||
@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
shopt -s failglob
|
||||
|
||||
pushd tmp
|
||||
|
||||
cat interrupted_dirs.tar.gz.part*.folder/interrupted_dirs.tar.gz.part* > interrupted_dirs.tar.gz
|
||||
|
||||
rm -rf interrupted_dirs.tar.gz.part*.folder
|
||||
|
||||
tar -xaf interrupted_dirs.tar.gz
|
||||
|
||||
rm interrupted_dirs.tar.gz
|
||||
|
||||
popd
|
||||
@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
echo "Interrupting build container..."
|
||||
|
||||
# Make sure tor-browser-build knows why the build stopped.
|
||||
touch tmp/interrupted
|
||||
|
||||
# Send SIGINT to all processes inside the container.
|
||||
kill -s SIGINT $(./tools/container-pids.sh)
|
||||
|
||||
echo "Interrupted!"
|
||||
@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# https://unix.stackexchange.com/a/299198
|
||||
descendent_pids() {
|
||||
pids=$(pgrep -P $1)
|
||||
echo $pids
|
||||
for pid in $pids; do
|
||||
descendent_pids $pid
|
||||
done
|
||||
}
|
||||
|
||||
build_pid=$(pgrep -f '\./build')
|
||||
echo $build_pid
|
||||
descendent_pids $build_pid
|
||||
@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euxo pipefail
|
||||
shopt -s nullglob globstar
|
||||
|
||||
cat tor-browser-build/rbm.conf | sed "s/torbrowser/ncdns/g" > rbm.conf
|
||||
@ -0,0 +1,14 @@
|
||||
|
||||
var:
|
||||
### The clean configuration is used by the cleaning script to find the
|
||||
### branches and build targets you are using, to compute the list of
|
||||
### files that should be kept.
|
||||
###
|
||||
### If you only do alpha builds for all platforms, you can use the
|
||||
### following configuration:
|
||||
clean:
|
||||
HEAD:
|
||||
- project: release
|
||||
target:
|
||||
- CHANNEL
|
||||
- ncdns-all
|
||||
@ -1 +1 @@
|
||||
Subproject commit 637e37bff412b587970791dafdb3174eb359d6b3
|
||||
Subproject commit 896b7c34db546b55bb466e24e87b01a7eec434e4
|
||||
Loading…
Reference in New Issue