From 23450a302712e5657193c3a6453f83ce96f2d150 Mon Sep 17 00:00:00 2001 From: gdm85 Date: Wed, 11 Jun 2014 00:26:09 +0200 Subject: [PATCH] * parallelize wget downloads of inputs * removed extra environment cleanup from bash_profile and added proper gitian-builder patch * use gitian-host-vms for gitian-bitcoin-host * fixed issue at stopping container after generation of base VMs --- docker/gitian-bitcoin-host/Dockerfile | 4 +-- docker/gitian-bitcoin-host/build-bitcoin.sh | 30 ++++++++++++--------- docker/gitian-bitcoin-host/gitian.patch | 15 ++++++++++- docker/gitian-host/bash_profile | 7 ----- docker/scripts/create-gitian-host.sh | 3 ++- 5 files changed, 35 insertions(+), 24 deletions(-) diff --git a/docker/gitian-bitcoin-host/Dockerfile b/docker/gitian-bitcoin-host/Dockerfile index c8ded7d..4243d82 100644 --- a/docker/gitian-bitcoin-host/Dockerfile +++ b/docker/gitian-bitcoin-host/Dockerfile @@ -3,7 +3,7 @@ ## VERSION 0.1.0 ## -FROM gdm85/gitian-host +FROM gdm85/gitian-host-vms MAINTAINER Giuseppe Mazzotta "gdm85@users.noreply.github.com" @@ -12,7 +12,7 @@ RUN apt-get install -y patch ADD gitian.patch /home/debian/ ## patch to allow paralle creation of VMs -RUN cd /home/debian/gitian-builder && patch -p1 < ../gitian.patch +RUN cd /home/debian/gitian-builder && patch -F3 -p1 < ../gitian.patch ADD input-sources/ /home/debian/input-sources/ ADD build-bitcoin.sh /home/debian/ diff --git a/docker/gitian-bitcoin-host/build-bitcoin.sh b/docker/gitian-bitcoin-host/build-bitcoin.sh index 7cef303..2dbd426 100755 --- a/docker/gitian-bitcoin-host/build-bitcoin.sh +++ b/docker/gitian-bitcoin-host/build-bitcoin.sh @@ -1,4 +1,10 @@ #!/bin/bash +## automatic Gitian build of bitcoin +## @author gdm85 +## @version 0.3.0 +## see also https://github.com/gdm85/tenku/blob/master/docker/gitian-bitcoin-host/ +## +# if [[ ! $# -eq 1 ]]; then echo "Please specify version" 1>&2 @@ -11,8 +17,7 @@ if [ ! -d bitcoin ]; then git clone https://github.com/bitcoin/bitcoin.git || exit $? fi cd bitcoin && \ -git checkout v${VERSION} || exit $? - +git checkout v${VERSION} && \ cd ../gitian-builder && \ mkdir -p inputs && cd inputs/ || exit $? @@ -22,17 +27,16 @@ while read -r URL FNAME; do if [ -z "$URL" ]; then continue fi - wget --continue --no-check-certificate "$URL" -O "$FNAME" || exit $? -done < ../../input-sources/${VERSION}.txt || exit $? + echo "wget -q --continue --no-check-certificate '$URL' -O '$FNAME'" +done < ../../input-sources/${VERSION}.txt | parallel -j10 || exit $? ## verify that all sources are correct before continuing -md5sum -c < ../../input-sources/${VERSION}.txt.md5 || exit $? - -cd .. -./bin/gbuild ../bitcoin/contrib/gitian-descriptors/boost-linux.yml || exit $? -mv build/out/boost-*.zip inputs/ -./bin/gbuild ../bitcoin/contrib/gitian-descriptors/deps-linux.yml || exit $? -mv build/out/bitcoin-deps-*.zip inputs/ -./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml || exit $? -echo "Completed successfully." +md5sum -c < ../../input-sources/${VERSION}.txt.md5 && \ +cd .. && \ +./bin/gbuild ../bitcoin/contrib/gitian-descriptors/boost-linux.yml && \ +mv build/out/boost-*.zip inputs/ && \ +./bin/gbuild ../bitcoin/contrib/gitian-descriptors/deps-linux.yml && \ +mv build/out/bitcoin-deps-*.zip inputs/ && \ +./bin/gbuild --commit bitcoin=v${VERSION} ../bitcoin/contrib/gitian-descriptors/gitian-linux.yml && \ +echo "Completed successfully." && \ echo "The output files are in: gitian-builder/build/out/" diff --git a/docker/gitian-bitcoin-host/gitian.patch b/docker/gitian-bitcoin-host/gitian.patch index 9d3f652..ffcecb7 100644 --- a/docker/gitian-bitcoin-host/gitian.patch +++ b/docker/gitian-bitcoin-host/gitian.patch @@ -1,9 +1,22 @@ +diff --git a/bin/gbuild b/bin/gbuild +index 4e7839f..d1601d3 100755 +--- a/bin/gbuild ++++ b/bin/gbuild +@@ -94,6 +94,8 @@ def build_one_configuration(suite, arch, build_desc, reference_datetime) + File.open("var/build-script", "w") do |script| + script.puts "#!/bin/bash" + script.puts "set -e" ++ script.puts "export LANG='en_US.UTF-8'" ++ script.puts "export LC_ALL='en_US.UTF-8'" + script.puts "export OUTDIR=$HOME/out" + script.puts "GBUILD_BITS=#{bits}" + script.puts "MAKEOPTS=(-j#{@options[:num_procs]})" diff --git a/bin/make-base-vm b/bin/make-base-vm index c1920f3..8a44f13 100755 --- a/bin/make-base-vm +++ b/bin/make-base-vm @@ -109,16 +109,3 @@ rm -rf $OUT - sudo vmbuilder kvm ubuntu --rootsize 10240 --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_dsa.pub --ssh-user-key=var/id_dsa.pub --mirror=$MIRROR --secu + sudo vmbuilder kvm ubuntu --rootsize 10240 --arch=$ARCH --suite=$SUITE --addpkg=$addpkg --removepkg=$removepkg --ssh-key=var/id_dsa.pub --ssh-user-key=var mv $OUT/*.qcow2 $OUT.qcow2 rm -rf $OUT - diff --git a/docker/gitian-host/bash_profile b/docker/gitian-host/bash_profile index 2bcca5e..361da3f 100644 --- a/docker/gitian-host/bash_profile +++ b/docker/gitian-host/bash_profile @@ -1,10 +1,3 @@ export USE_LXC=1 export GITIAN_HOST_IP=10.0.3.2 export LXC_GUEST_IP=10.0.3.5 - -## see bugs https://github.com/devrandom/gitian-builder/issues/56 and https://bugzilla.mindrot.org/show_bug.cgi?id=1285 -for ENV in $(env | grep ^LC | awk -F= '{ print $1 }'); do - unset $ENV -done - -export LANG=en_US.UTF-8 diff --git a/docker/scripts/create-gitian-host.sh b/docker/scripts/create-gitian-host.sh index bba28b1..a4e22fc 100755 --- a/docker/scripts/create-gitian-host.sh +++ b/docker/scripts/create-gitian-host.sh @@ -37,7 +37,8 @@ echo "Now building base VMs" && \ IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \ wait_for_ssh $IP 10 && \ ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP ./build-base-vms.sh && \ -docker kill $CID && \ +docker stop $CID && \ +docker wait $CID && \ docker commit $CID gdm85/gitian-host-vms && \ docker rm $CID && \ echo "Gitian host images created successfully!" && \