From 79c575a8ef098875ecec98617572d809d36f55c4 Mon Sep 17 00:00:00 2001
From: Jeremy Rand <jeremyrand@airmail.cc>
Date: Wed, 17 Feb 2021 03:00:05 +0000
Subject: [PATCH] Add skeleton of Cirrus rbm builds

---
 .cirrus.yml                   | 82 +++++++++++++++++++++++++++++++++++
 tools/cirrus_build_project.sh | 73 +++++++++++++++++++++++++++++++
 tools/cirrus_gen_yml.sh       | 73 +++++++++++++++++++++++++++++++
 3 files changed, 228 insertions(+)
 create mode 100644 .cirrus.yml
 create mode 100755 tools/cirrus_build_project.sh
 create mode 100755 tools/cirrus_gen_yml.sh

diff --git a/.cirrus.yml b/.cirrus.yml
new file mode 100644
index 0000000..5ae250f
--- /dev/null
+++ b/.cirrus.yml
@@ -0,0 +1,82 @@
+nightly_linux_x86_64_download_docker_builder:
+  timeout_in: 120m
+  out_nightly_linux_x86_64_cache:
+    folder: out
+    fingerprint_script:
+      - "echo out_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p out"
+  git_nightly_linux_x86_64_cache:
+    folder: git_clones
+    fingerprint_script:
+      - "echo git_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p git_clones"
+  build_script:
+    - "./tools/cirrus_build_project.sh ncdns nightly linux x86_64 0"
+
+nightly_linux_x86_64_ncdns_docker_builder:
+  timeout_in: 120m
+  out_nightly_linux_x86_64_cache:
+    folder: out
+    fingerprint_script:
+      - "echo out_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p out"
+  git_nightly_linux_x86_64_cache:
+    folder: git_clones
+    fingerprint_script:
+      - "echo git_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p git_clones"
+  build_script:
+    - "./tools/cirrus_build_project.sh ncdns nightly linux x86_64 1"
+  depends_on:
+    - "nightly_linux_x86_64_download"
+
+nightly_linux_x86_64_ncp11_docker_builder:
+  timeout_in: 120m
+  out_nightly_linux_x86_64_cache:
+    folder: out
+    fingerprint_script:
+      - "echo out_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p out"
+  git_nightly_linux_x86_64_cache:
+    folder: git_clones
+    fingerprint_script:
+      - "echo git_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p git_clones"
+  build_script:
+    - "./tools/cirrus_build_project.sh ncp11 nightly linux x86_64 1"
+  depends_on:
+    - "nightly_linux_x86_64_ncdns"
+
+nightly_linux_x86_64_ncprop279_docker_builder:
+  timeout_in: 120m
+  out_nightly_linux_x86_64_cache:
+    folder: out
+    fingerprint_script:
+      - "echo out_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p out"
+  git_nightly_linux_x86_64_cache:
+    folder: git_clones
+    fingerprint_script:
+      - "echo git_nightly_linux_x86_64"
+    reupload_on_changes: true
+    populate_script:
+      - "mkdir -p git_clones"
+  build_script:
+    - "./tools/cirrus_build_project.sh ncprop279 nightly linux x86_64 1"
+  depends_on:
+    - "nightly_linux_x86_64_ncp11"
+
diff --git a/tools/cirrus_build_project.sh b/tools/cirrus_build_project.sh
new file mode 100755
index 0000000..c96ee17
--- /dev/null
+++ b/tools/cirrus_build_project.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+shopt -s nullglob globstar
+
+PROJECT=$1
+CHANNEL=$2
+OS=$3
+ARCH=$4
+SHOULD_BUILD=$5
+
+echo "Checking VM specs..."
+cat /etc/*-release
+df -h
+lscpu
+free -m
+
+echo "Installing rbm deps..."
+apt-get install -y libyaml-libyaml-perl libtemplate-perl libio-handle-util-perl libio-all-perl libio-captureoutput-perl libjson-perl libpath-tiny-perl libstring-shellquote-perl libsort-versions-perl libdigest-sha-perl libdata-uuid-perl libdata-dump-perl libfile-copy-recursive-perl libfile-slurp-perl git runc
+
+echo "Pulling rbm..."
+make submodule-update
+
+echo "Moving caches..."
+if [[ -e "./fonts/.git" ]]; then
+    echo "git_clones/fonts was cached, moving it to the right place..."
+    mv ./fonts ./git_clones/fonts
+else
+    echo "git_clones/fonts was not cached."
+    rm -rf ./fonts ./git_clones/fonts
+fi
+ls ./git_clones
+
+echo "Checking if project is cached..."
+OUTDIR="$(./rbm/rbm showconf $PROJECT output_dir --target $CHANNEL --target ncdns-$OS-$ARCH)"
+OUTFILE="$(./rbm/rbm showconf $PROJECT filename --target $CHANNEL --target ncdns-$OS-$ARCH)"
+if [[ -e "$OUTDIR/$OUTFILE" ]]; then
+    echo "Project cache hit, skipping build."
+    SHOULD_BUILD=0
+else
+    echo "Project cache miss, proceeding with build."
+fi
+
+# VM has 12 GB of free RAM.  Assuming each of the 4 logical cores takes 1 GB
+# during build, that leaves us with 8 GB of unutilized RAM.  Alas, I'm not sure
+# that's enough, so this isn't enabled right now.
+#echo "Mounting tmpfs..."
+#mount -t tmpfs -o size=8G,nr_inodes=40k,mode=1777 tmpfs ./tmp
+#df -h
+
+if [[ "$SHOULD_BUILD" -eq 1 ]]; then
+    echo "Building project..."
+    ./rbm/rbm build "$PROJECT" --target "$CHANNEL" --target ncdns-"$OS"-"$ARCH"
+else
+    #echo "This is a cache-only task, skipping build."
+    echo "Skipping build."
+fi
+
+echo "Moving caches..."
+if [[ -e "git_clones/fonts" ]]; then
+    echo "git_clones/fonts is ready to be cached, moving it to the right place..."
+    mv git_clones/fonts ./
+else
+    echo "git_clones/fonts is not ready to be not cached."
+    mkdir -p ./fonts
+fi
+
+# The cache has a size limit, so we need to clean useless data from it.  The
+# container-images are very large and seem to be fairly harmless to remove.
+# Maybe later if we have more pressure to shrink, we could remove the
+# debootstrap-images too.
+echo "Cleaning cache..."
+rm -rfv out/container-image
diff --git a/tools/cirrus_gen_yml.sh b/tools/cirrus_gen_yml.sh
new file mode 100755
index 0000000..9e0709a
--- /dev/null
+++ b/tools/cirrus_gen_yml.sh
@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+shopt -s nullglob globstar
+
+(
+#for CHANNEL in release nightly; do
+for CHANNEL in nightly; do
+    OS=linux
+    ARCH=x86_64
+
+    # Pre-download tarballs and Git repos
+    echo "${CHANNEL}_${OS}_${ARCH}_download_docker_builder:
+  timeout_in: 120m
+  out_${CHANNEL}_${OS}_${ARCH}_cache:
+    folder: out
+    fingerprint_script:
+      - \"echo out_${CHANNEL}_${OS}_${ARCH}\"
+    reupload_on_changes: true
+    populate_script:
+      - \"mkdir -p out\"
+  git_${CHANNEL}_${OS}_${ARCH}_cache:
+    folder: git_clones
+    fingerprint_script:
+      - \"echo git_${CHANNEL}_${OS}_${ARCH}\"
+    reupload_on_changes: true
+    populate_script:
+      - \"mkdir -p git_clones\"
+  build_script:
+    - \"./tools/cirrus_build_project.sh ncdns ${CHANNEL} ${OS} ${ARCH} 0\""
+    echo ""
+
+    # TODO fine-tune this list
+    for PROJECT in ncdns ncp11 ncprop279; do
+        echo "${CHANNEL}_${OS}_${ARCH}_${PROJECT}_docker_builder:
+  timeout_in: 120m
+  out_${CHANNEL}_${OS}_${ARCH}_cache:
+    folder: out
+    fingerprint_script:
+      - \"echo out_${CHANNEL}_${OS}_${ARCH}\"
+    reupload_on_changes: true
+    populate_script:
+      - \"mkdir -p out\"
+  git_${CHANNEL}_${OS}_${ARCH}_cache:
+    folder: git_clones
+    fingerprint_script:
+      - \"echo git_${CHANNEL}_${OS}_${ARCH}\"
+    reupload_on_changes: true
+    populate_script:
+      - \"mkdir -p git_clones\"
+  build_script:
+    - \"./tools/cirrus_build_project.sh ${PROJECT} ${CHANNEL} ${OS} ${ARCH} 1\""
+
+        # Depend on previous project
+        if [[ "$PROJECT" == "ncdns" ]]; then
+            echo "  depends_on:
+    - \"${CHANNEL}_${OS}_${ARCH}_download\""
+        else
+            echo "  depends_on:
+    - \"${CHANNEL}_${OS}_${ARCH}_${PREV_PROJECT}\""
+        fi
+
+        PREV_PROJECT="$PROJECT"
+        echo ""
+    done
+done
+) > .cirrus.yml
+
+# Timeout issues?
+# Might want to increase the timeout -- but we're already using the 2 hour max.
+# Might want to bump the CPU count -- but that's blocked by cirrus-ci-docs issue #741.
+# Might want to split into smaller project sets.
+# What is the CPU count limit?  "Linux Containers" docs say 8.0 CPU and 24 GB RAM; "FAQ" says 16.0 CPU.  docker_builder VM's are really 4.0 CPU and 15 GB RAM (12 GB of which is unused by the OS).