blob42
/
tenku
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Completed automation for unattended gitian bitcoin builds

Browse Source
master
gdm85 9 years ago
parent 8f79fccb14
commit 7bd75cc4b4
5 changed files with 82 additions and 86 deletions
Show Stats Download Patch File Download Diff File

5
docker/gitian-bitcoin-host/bin/build-bitcoin.sh
Unescape Escape View File

@ -21,6 +21,7 @@ function verlte() {
[ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
}
NPROC=$(nproc) && \
cd gitian-builder && \
mkdir -p inputs && \
cd .. || exit $?
@ -60,7 +61,7 @@ if ! verlte 0.10.0rc1 ${VERSION}; then
cd .. || exit $?
for DESC in $DESCRIPTORS; do
./bin/gbuild ../bitcoin/contrib/gitian-descriptors/${DESC}.yml && \
./bin/gbuild -j$NPROC ../bitcoin/contrib/gitian-descriptors/${DESC}.yml && \
mv -v $(find build/out -type f -name '*gz' -o -name '*.zip') inputs/ || exit $?
done
else
@ -74,7 +75,7 @@ fi
## proceed to build of each of the specified gitian descriptors
cd gitian-builder || exit $?
for DESC in $@; do
./bin/gbuild --commit bitcoin=v$VERSION -u bitcoin=$CLONE "$CLONE/contrib/gitian-descriptors/gitian-${DESC}.yml" || exit $?
./bin/gbuild -j$NPROC --commit bitcoin=v$VERSION -u bitcoin=$CLONE "$CLONE/contrib/gitian-descriptors/gitian-${DESC}.yml" || exit $?
done
echo "Build completed successfully, output files are in: ~/gitian-builder/build/out/"

7
docker/gitian-host/Dockerfile
Unescape Escape View File

@ -12,11 +12,10 @@ FROM gdm85/wheezy
MAINTAINER Giuseppe Mazzotta "gdm85@users.noreply.github.com"
ENV TERM xterm
ENV DEBIAN_FRONTEND noninteractive
## some core packages
RUN apt-get update && apt-get install -y htop tmux less locales openssh-server
RUN apt-get update && apt-get install -y aptitude
RUN aptitude update && aptitude safe-upgrade -y
RUN aptitude install htop tmux less locales openssh-server -y
## these 2 files can be customized
COPY etc/timezone /etc/

50
docker/gitian-host/bin/build-base-vms.sh
Unescape Escape View File

@ -1,37 +1,47 @@
#!/bin/bash
set -e
source ~/.bash_profile
if [ $# -eq 0 ]; then
echo "Please specify: [i386|amd64]" 1>&2
exit 1
fi
cd gitian-builder
mkdir -p var
if [ ! -e var/id_dsa ]; then
ssh-keygen -t dsa -f var/id_dsa -N ""
if [ -z "$USE_LXC" ]; then
echo "Environment variables not correctly setup (source .bash_profile?)" 1>&2
exit 2
fi
export MIRROR_HOST=$GITIAN_HOST_IP
SUITE=precise
cd gitian-builder && \
mkdir -p var || exit $?
if [ ! -e var/id_dsa ]; then
ssh-keygen -t dsa -f var/id_dsa -N ""
fi
## build both VMs in parallel
echo "Now building i386 and amd64 VMs..."
echo -e "MIRROR_HOST=$GITIAN_HOST_IP bin/make-base-vm --lxc --arch i386 --suite $SUITE\nMIRROR_HOST=$GITIAN_HOST_IP bin/make-base-vm --lxc --arch amd64 --suite $SUITE" \
| parallel -j2 || exit $?
for TYPE in "$@"; do
echo -e "MIRROR_HOST=$GITIAN_HOST_IP bin/make-base-vm --lxc --arch $TYPE --suite $SUITE"
done | parallel -j$# || exit $?
## this function corresponds to part removed from gbuild via custom patch
function ext_partition() {
local OUT=$1
echo Extracting $OUT partition for lxc
qemu-img convert $OUT.qcow2 $OUT.raw
loop=`sudo kpartx -av $OUT.raw|sed -n '/loop.p1/{s/.*loop\(.\)p1.*/\1/;p}'`
sudo cp --sparse=always /dev/mapper/loop${loop}p1 $OUT
sudo chown $USER $OUT
echo "Extracting $OUT partition for lxc" && \
qemu-img convert $OUT.qcow2 $OUT.raw && \
loop=`sudo kpartx -av $OUT.raw|sed -n '/loop.p1/{s/.*loop\(.\)p1.*/\1/;p}'` || return $?
sudo cp --sparse=always /dev/mapper/loop${loop}p1 $OUT && \
sudo chown $USER $OUT || return $?
## following 2 lines are a sloppy hack to an unknown problem with kpartx
sudo sync
sleep 5
sudo sync && \
sleep 5 || return $?
## these are silenced because if former fails, second doesn't and viceversa
sudo kpartx -d /dev/loop$loop 2>/dev/null
sudo rm /dev/mapper/loop${loop}p1 2>/dev/null
sudo kpartx -d /dev/loop$loop 2>/dev/null && \
sudo rm /dev/mapper/loop${loop}p1 2>/dev/null && \
rm -f $OUT.raw
}
ext_partition base-${SUITE}-i386 && \
ext_partition base-${SUITE}-amd64 || exit $?
for TYPE in "$@"; do
ext_partition base-${SUITE}-${TYPE} || exit $?
done

70
docker/scripts/bitcoin-gitian-build.sh
Unescape Escape View File

@ -1,10 +1,24 @@
#!/bin/bash
## bitcoin-gitian-build.sh
##
## @author gdm85
##
## Automatically build latest version of Bitcoin Core using
## Docker containers (LXC) + KVM.
##
## User can specify target operative systems as arguments.
##
#
SCRIPTS=$(dirname $(readlink -m $0)) || exit $?
if [ $# -lt 1 ]; then
echo "Usage: gitian-build.sh linux [win] [osx] [...]" 1>&2
exit 1
fi
## identify a CLI tool to run commands in parallel
## coshell is preferred
PARALLEL=""
if type coshell 2>/dev/null >/dev/null; then
PARALLEL="coshell"
@ -17,40 +31,23 @@ else
fi
fi
## retrieve latest tagged release/release candidate
set -o pipefail && \
MOSTRECENT="$(curl -s https://api.github.com/repos/bitcoin/bitcoin/tags | jq -r '.[0].name' | awk '{ print substr($0, 2) }')" || exit $?
## run all necessary containers, detached
## setup proper volumes for input/output collection
function run_all() {
local OS
local SRCV="/home/debian/gitian-build/inputs"
local DSTV="/home/debian/gitian-build/build/out"
for OS in "$@"; do
echo "docker run -d --privileged gdm85/gitian-bitcoin-host"
mkdir -p "$SCRIPTS/cache/${OS}-inputs" "$SCRIPTS/built/${OS}" && \
echo "docker run -d --privileged -v $SCRIPTS/cache/${OS}-inputs:${SRCV} -v $SCRIPTS/built/${OS}:${DSTV} gdm85/gitian-bitcoin-host" || return $?
done | $PARALLEL
}
## run a simple test to detect if SSH works
function loop_wait_all() {
local RETRIES="$1"
shift
while [ $RETRIES -gt 0 ]; do
wait_all "$@" && break
sleep 1
let RETRIES-=1
done
return 0
}
function wait_all() {
local CID
local IP
for CID in "$@"; do
IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \
echo "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP true" || return $?
done | $PARALLEL 2>/dev/null
}
function build_all() {
local ALL=($@)
local LEN=$(($#/2))
@ -63,34 +60,23 @@ function build_all() {
local I=0
for CID in $CREATED; do
OS=${OSES[$I]}
# IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \
# echo -n "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP " && \
# echo "./build-bitcoin.sh $MOSTRECENT ${OS}" || return $?
echo "docker-enter $CID su -c 'cd /home/debian && ./build-bitcoin.sh $MOSTRECENT ${OS}' debian"
echo "docker exec $CID su -c 'cd /home/debian && source .bash_profile && ./build-bitcoin.sh $MOSTRECENT ${OS}' debian"
let I+=1
done | $PARALLEL
}
function copy_all() {
local OS
for OS in "$@"; do
echo "docker cp ${CID}:/home/debian/gitian-build/build/out built-${OS}"
done | $PARALLEL
}
CREATED="$(run_all $@ | tr '\n' ' ')" && \
echo loop_wait_all 5 $CREATED && \
echo "Containers are online: $CREATED, building bitcoin v$MOSTRECENT" && \
echo "Building bitcoin v$MOSTRECENT on containers $CREATED" && \
build_all $CREATED $@ && \
copy_all $CREATED
echo "Build results are available in '$SCRIPTS/built/'"
RV=$?
## cleanup
#echo "Cleaning up created containers..."
#for CID in $CREATED; do
# docker stop $CID
# docker rm $CID
#done
echo "Cleaning up created containers..."
for CID in $CREATED; do
docker stop $CID
docker rm $CID
done
## return build exit code
exit $RV

36
docker/scripts/create-gitian-host.sh
Unescape Escape View File

@ -2,23 +2,6 @@
BASENAME=$(dirname $(readlink -m $0))
cd $BASENAME/../gitian-host || exit $?
if [ ! -f authorized_keys ]; then
echo "No authorized_keys file found in $PWD"
if [ -f ~/.ssh/id_rsa.pub ]; then
echo -n "Do you want to use ~/.ssh/id_rsa.pub? (y/n) "
read -r ANSWER
if [[ "$ANSWER" == "y" ]]; then
cp -v ~/.ssh/id_rsa.pub authorized_keys || exit $?
else
exit 1
fi
else
exit 1
fi
fi
function wait_for_ssh() {
local IP="$1"
local SECS="$2"
@ -37,13 +20,30 @@ function wait_remove() {
done
}
cd $BASENAME/../gitian-host || exit $?
if [ ! -f authorized_keys ]; then
echo "No authorized_keys file found in $PWD"
if [ -f ~/.ssh/id_rsa.pub ]; then
echo -n "Do you want to use ~/.ssh/id_rsa.pub? (y/n) "
read -r ANSWER
if [[ "$ANSWER" == "y" ]]; then
cp -v ~/.ssh/id_rsa.pub authorized_keys || exit $?
else
exit 1
fi
else
exit 1
fi
fi
##NOTE: can leave behind a running container of gitian-host
docker build --tag=gdm85/gitian-host . && \
CID=$(docker run -d --privileged gdm85/gitian-host) && \
IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \
wait_for_ssh "$IP" 10 && \
echo "$CID is now online ($IP), building base VMs on it" && \
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP ./build-base-vms.sh && \
ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP bash -c 'cd /home/debian && source ./.bash_profile && ./build-base-vms.sh amd64' && \
docker kill $CID && \
docker wait $CID && \
docker commit $CID gdm85/gitian-host-vms && \

Write Preview
Loading…
Cancel
Save