diff --git a/docker/gitian-bitcoin-host/bin/build-bitcoin.sh b/docker/gitian-bitcoin-host/bin/build-bitcoin.sh index 14738af..665cf68 100755 --- a/docker/gitian-bitcoin-host/bin/build-bitcoin.sh +++ b/docker/gitian-bitcoin-host/bin/build-bitcoin.sh @@ -21,6 +21,7 @@ function verlte() { [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] } +NPROC=$(nproc) && \ cd gitian-builder && \ mkdir -p inputs && \ cd .. || exit $? @@ -60,7 +61,7 @@ if ! verlte 0.10.0rc1 ${VERSION}; then cd .. || exit $? for DESC in $DESCRIPTORS; do - ./bin/gbuild ../bitcoin/contrib/gitian-descriptors/${DESC}.yml && \ + ./bin/gbuild -j$NPROC ../bitcoin/contrib/gitian-descriptors/${DESC}.yml && \ mv -v $(find build/out -type f -name '*gz' -o -name '*.zip') inputs/ || exit $? done else @@ -74,7 +75,7 @@ fi ## proceed to build of each of the specified gitian descriptors cd gitian-builder || exit $? for DESC in $@; do - ./bin/gbuild --commit bitcoin=v$VERSION -u bitcoin=$CLONE "$CLONE/contrib/gitian-descriptors/gitian-${DESC}.yml" || exit $? + ./bin/gbuild -j$NPROC --commit bitcoin=v$VERSION -u bitcoin=$CLONE "$CLONE/contrib/gitian-descriptors/gitian-${DESC}.yml" || exit $? done echo "Build completed successfully, output files are in: ~/gitian-builder/build/out/" diff --git a/docker/gitian-host/Dockerfile b/docker/gitian-host/Dockerfile index 689bbb0..41ff590 100644 --- a/docker/gitian-host/Dockerfile +++ b/docker/gitian-host/Dockerfile @@ -12,11 +12,10 @@ FROM gdm85/wheezy MAINTAINER Giuseppe Mazzotta "gdm85@users.noreply.github.com" -ENV TERM xterm -ENV DEBIAN_FRONTEND noninteractive - ## some core packages -RUN apt-get update && apt-get install -y htop tmux less locales openssh-server +RUN apt-get update && apt-get install -y aptitude +RUN aptitude update && aptitude safe-upgrade -y +RUN aptitude install htop tmux less locales openssh-server -y ## these 2 files can be customized COPY etc/timezone /etc/ diff --git a/docker/gitian-host/bin/build-base-vms.sh b/docker/gitian-host/bin/build-base-vms.sh index c45536a..59387e3 100755 --- a/docker/gitian-host/bin/build-base-vms.sh +++ b/docker/gitian-host/bin/build-base-vms.sh @@ -1,37 +1,47 @@ #!/bin/bash -set -e -source ~/.bash_profile +if [ $# -eq 0 ]; then + echo "Please specify: [i386|amd64]" 1>&2 + exit 1 +fi -cd gitian-builder -mkdir -p var -if [ ! -e var/id_dsa ]; then - ssh-keygen -t dsa -f var/id_dsa -N "" +if [ -z "$USE_LXC" ]; then + echo "Environment variables not correctly setup (source .bash_profile?)" 1>&2 + exit 2 fi export MIRROR_HOST=$GITIAN_HOST_IP SUITE=precise +cd gitian-builder && \ +mkdir -p var || exit $? + +if [ ! -e var/id_dsa ]; then + ssh-keygen -t dsa -f var/id_dsa -N "" +fi + ## build both VMs in parallel -echo "Now building i386 and amd64 VMs..." -echo -e "MIRROR_HOST=$GITIAN_HOST_IP bin/make-base-vm --lxc --arch i386 --suite $SUITE\nMIRROR_HOST=$GITIAN_HOST_IP bin/make-base-vm --lxc --arch amd64 --suite $SUITE" \ - | parallel -j2 || exit $? +for TYPE in "$@"; do + echo -e "MIRROR_HOST=$GITIAN_HOST_IP bin/make-base-vm --lxc --arch $TYPE --suite $SUITE" +done | parallel -j$# || exit $? +## this function corresponds to part removed from gbuild via custom patch function ext_partition() { local OUT=$1 - echo Extracting $OUT partition for lxc - qemu-img convert $OUT.qcow2 $OUT.raw - loop=`sudo kpartx -av $OUT.raw|sed -n '/loop.p1/{s/.*loop\(.\)p1.*/\1/;p}'` - sudo cp --sparse=always /dev/mapper/loop${loop}p1 $OUT - sudo chown $USER $OUT + echo "Extracting $OUT partition for lxc" && \ + qemu-img convert $OUT.qcow2 $OUT.raw && \ + loop=`sudo kpartx -av $OUT.raw|sed -n '/loop.p1/{s/.*loop\(.\)p1.*/\1/;p}'` || return $? + sudo cp --sparse=always /dev/mapper/loop${loop}p1 $OUT && \ + sudo chown $USER $OUT || return $? ## following 2 lines are a sloppy hack to an unknown problem with kpartx - sudo sync - sleep 5 + sudo sync && \ + sleep 5 || return $? ## these are silenced because if former fails, second doesn't and viceversa - sudo kpartx -d /dev/loop$loop 2>/dev/null - sudo rm /dev/mapper/loop${loop}p1 2>/dev/null + sudo kpartx -d /dev/loop$loop 2>/dev/null && \ + sudo rm /dev/mapper/loop${loop}p1 2>/dev/null && \ rm -f $OUT.raw } -ext_partition base-${SUITE}-i386 && \ -ext_partition base-${SUITE}-amd64 || exit $? +for TYPE in "$@"; do + ext_partition base-${SUITE}-${TYPE} || exit $? +done diff --git a/docker/scripts/bitcoin-gitian-build.sh b/docker/scripts/bitcoin-gitian-build.sh index 4822ea3..c2cfb34 100755 --- a/docker/scripts/bitcoin-gitian-build.sh +++ b/docker/scripts/bitcoin-gitian-build.sh @@ -1,10 +1,24 @@ #!/bin/bash +## bitcoin-gitian-build.sh +## +## @author gdm85 +## +## Automatically build latest version of Bitcoin Core using +## Docker containers (LXC) + KVM. +## +## User can specify target operative systems as arguments. +## +# + +SCRIPTS=$(dirname $(readlink -m $0)) || exit $? if [ $# -lt 1 ]; then echo "Usage: gitian-build.sh linux [win] [osx] [...]" 1>&2 exit 1 fi +## identify a CLI tool to run commands in parallel +## coshell is preferred PARALLEL="" if type coshell 2>/dev/null >/dev/null; then PARALLEL="coshell" @@ -17,40 +31,23 @@ else fi fi +## retrieve latest tagged release/release candidate set -o pipefail && \ MOSTRECENT="$(curl -s https://api.github.com/repos/bitcoin/bitcoin/tags | jq -r '.[0].name' | awk '{ print substr($0, 2) }')" || exit $? ## run all necessary containers, detached +## setup proper volumes for input/output collection function run_all() { local OS + local SRCV="/home/debian/gitian-build/inputs" + local DSTV="/home/debian/gitian-build/build/out" for OS in "$@"; do - echo "docker run -d --privileged gdm85/gitian-bitcoin-host" + mkdir -p "$SCRIPTS/cache/${OS}-inputs" "$SCRIPTS/built/${OS}" && \ + echo "docker run -d --privileged -v $SCRIPTS/cache/${OS}-inputs:${SRCV} -v $SCRIPTS/built/${OS}:${DSTV} gdm85/gitian-bitcoin-host" || return $? done | $PARALLEL } -## run a simple test to detect if SSH works -function loop_wait_all() { - local RETRIES="$1" - shift - while [ $RETRIES -gt 0 ]; do - wait_all "$@" && break - sleep 1 - let RETRIES-=1 - done - return 0 -} - -function wait_all() { - local CID - local IP - - for CID in "$@"; do - IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \ - echo "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP true" || return $? - done | $PARALLEL 2>/dev/null -} - function build_all() { local ALL=($@) local LEN=$(($#/2)) @@ -63,34 +60,23 @@ function build_all() { local I=0 for CID in $CREATED; do OS=${OSES[$I]} -# IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \ -# echo -n "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP " && \ -# echo "./build-bitcoin.sh $MOSTRECENT ${OS}" || return $? - echo "docker-enter $CID su -c 'cd /home/debian && ./build-bitcoin.sh $MOSTRECENT ${OS}' debian" + echo "docker exec $CID su -c 'cd /home/debian && source .bash_profile && ./build-bitcoin.sh $MOSTRECENT ${OS}' debian" let I+=1 done | $PARALLEL } -function copy_all() { - local OS - for OS in "$@"; do - echo "docker cp ${CID}:/home/debian/gitian-build/build/out built-${OS}" - done | $PARALLEL -} - CREATED="$(run_all $@ | tr '\n' ' ')" && \ -echo loop_wait_all 5 $CREATED && \ -echo "Containers are online: $CREATED, building bitcoin v$MOSTRECENT" && \ +echo "Building bitcoin v$MOSTRECENT on containers $CREATED" && \ build_all $CREATED $@ && \ -copy_all $CREATED +echo "Build results are available in '$SCRIPTS/built/'" RV=$? ## cleanup -#echo "Cleaning up created containers..." -#for CID in $CREATED; do -# docker stop $CID -# docker rm $CID -#done +echo "Cleaning up created containers..." +for CID in $CREATED; do + docker stop $CID + docker rm $CID +done ## return build exit code exit $RV diff --git a/docker/scripts/create-gitian-host.sh b/docker/scripts/create-gitian-host.sh index 2bb71ab..3299a00 100755 --- a/docker/scripts/create-gitian-host.sh +++ b/docker/scripts/create-gitian-host.sh @@ -2,23 +2,6 @@ BASENAME=$(dirname $(readlink -m $0)) -cd $BASENAME/../gitian-host || exit $? - -if [ ! -f authorized_keys ]; then - echo "No authorized_keys file found in $PWD" - if [ -f ~/.ssh/id_rsa.pub ]; then - echo -n "Do you want to use ~/.ssh/id_rsa.pub? (y/n) " - read -r ANSWER - if [[ "$ANSWER" == "y" ]]; then - cp -v ~/.ssh/id_rsa.pub authorized_keys || exit $? - else - exit 1 - fi - else - exit 1 - fi -fi - function wait_for_ssh() { local IP="$1" local SECS="$2" @@ -37,13 +20,30 @@ function wait_remove() { done } +cd $BASENAME/../gitian-host || exit $? + +if [ ! -f authorized_keys ]; then + echo "No authorized_keys file found in $PWD" + if [ -f ~/.ssh/id_rsa.pub ]; then + echo -n "Do you want to use ~/.ssh/id_rsa.pub? (y/n) " + read -r ANSWER + if [[ "$ANSWER" == "y" ]]; then + cp -v ~/.ssh/id_rsa.pub authorized_keys || exit $? + else + exit 1 + fi + else + exit 1 + fi +fi + ##NOTE: can leave behind a running container of gitian-host docker build --tag=gdm85/gitian-host . && \ CID=$(docker run -d --privileged gdm85/gitian-host) && \ IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \ wait_for_ssh "$IP" 10 && \ echo "$CID is now online ($IP), building base VMs on it" && \ -ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP ./build-base-vms.sh && \ +ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP bash -c 'cd /home/debian && source ./.bash_profile && ./build-base-vms.sh amd64' && \ docker kill $CID && \ docker wait $CID && \ docker commit $CID gdm85/gitian-host-vms && \