From 05ee2604126c3da8cfc0177a2d2b9068084f077a Mon Sep 17 00:00:00 2001 From: gdm85 Date: Sat, 7 Jun 2014 12:22:30 +0200 Subject: [PATCH] * added generation of gitian-host-vms image * simplified necessary steps * fixed issue with parallel package missing --- docker/gitian-bitcoin-host/Dockerfile | 2 +- docker/gitian-bitcoin-host/README.md | 32 ++++----------- docker/gitian-bitcoin-host/build-bitcoin.sh | 6 ++- docker/gitian-host/Dockerfile | 2 +- docker/gitian-host/README.md | 44 +++++++++++++-------- docker/gitian-host/setup.sh | 11 ++++-- docker/scripts/create-gitian-host.sh | 22 ++++++++++- docker/scripts/spawn-gitian-host.sh | 2 +- 8 files changed, 70 insertions(+), 51 deletions(-) diff --git a/docker/gitian-bitcoin-host/Dockerfile b/docker/gitian-bitcoin-host/Dockerfile index 80eebf8..c8ded7d 100644 --- a/docker/gitian-bitcoin-host/Dockerfile +++ b/docker/gitian-bitcoin-host/Dockerfile @@ -7,7 +7,7 @@ FROM gdm85/gitian-host MAINTAINER Giuseppe Mazzotta "gdm85@users.noreply.github.com" -RUN apt-get install -y parallel patch +RUN apt-get install -y patch ADD gitian.patch /home/debian/ diff --git a/docker/gitian-bitcoin-host/README.md b/docker/gitian-bitcoin-host/README.md index c22e7ca..9364dcc 100644 --- a/docker/gitian-bitcoin-host/README.md +++ b/docker/gitian-bitcoin-host/README.md @@ -2,26 +2,14 @@ Building bitcoin with a gitian-builder Docker container ======================================================= This image allows automated gitian builds of bitcoin using a docker container. -Before proceeding make sure you have created the necessary *wheezy* and *gitian-host* images, see [these instructions](../gitian-host/README.md) for the creation of both. +Before proceeding make sure you have created the necessary *gdm85/wheezy*, *gdm85/gitian-host* and *gdm85/gitian-host-vms* images, see [these instructions](../gitian-host/README.md). -Afterwards you can create the *gitian-bitcoin-host* image by running [scripts/create-gitian-bitcoin-host.sh](../scripts/create-gitian-bitcoin-host.sh). +Afterwards you can create the *gdm85/gitian-bitcoin-host* image by running [scripts/create-gitian-bitcoin-host.sh](../scripts/create-gitian-bitcoin-host.sh). NOTE: this image currently supports only building of bitcoin 0.9.1, but it can be easily adapted to build other versions. You can submit the source lists for other versions as a patch or pull request, see directory [input-sources/](input-sources/) for currently available versions. -Preamble --------- - -It is **necessary** that before you using these scripts you read them and understand what they do. -Why? Because your goal is to create a gitian build (deterministic) that has not been tampered with. - -See also: -- https://gitian.org/ -- https://en.wikipedia.org/wiki/Web_of_trust -- http://www.dwheeler.com/trusting-trust/ -- https://www.debian.org/ -- https://www.docker.io/ -- http://www.ubuntu.com/ +Do not forget to read also the [Preamble here](../gitian-host/README.md#preamble] to correctly use Gitian builder and these provided scripts. Spawning a container -------------------- @@ -43,27 +31,23 @@ You can use this specific SSH command line to get a shell in the container and p Preparing the gitian environment -------------------------------- -First prepare the base VMs inside the gitian host container by running: - -- [./build-base-vms.sh](../gitian-host/build-base-vms.sh) - -This operation will take a while; afterwards you can proceed to building bitcoin with: +To initiate a gitian build of bitcoin you will run: - [./build-bitcoin.sh](build-bitcoin.sh) 0.9.1 -Notice the parameter 0.9.1, that is the version we are going to build. +Notice the parameter 0.9.1, that is the version we are going to build and must be available in [input-sources](input-sources/). [build-bitcoin.sh](build-bitcoin.sh) is a script that will download & build all the dependencies and then bitcoin itself, for both i386 and amd64 Linux architectures. Signing ------- -Now you have completed the build of bitcoin and only the signing part is left. -Before doing that, you can verify if signatures are matching with those of [other developers](https://github.com/bitcoin/gitian.sigs) by peeking inside *~/gitian.sigs* of the running container. +Once you have completed the build of bitcoin, you will be ready to perform the signing; before doing that you should verify that signatures are matching with those of [other developers](https://github.com/bitcoin/gitian.sigs) by peeking inside *~/gitian.sigs* of the running container. +Only the out_manifest signatures do matter for this purpose. In order to sign you have to either put your private key in the container's *~/.gnupg* or perform the signing externally, at your option. -If you have the private key in the container (also displayed by `gpg -K`), then you can use the [sign.sh](sign.sh) script that is already in the running container, otherwise +If you have the private key in the container (also displayed by `gpg -K`), then you can use the [sign.sh](sign.sh) script that is already provided, otherwise run it (with failure) and then copy the *~/gitian.sigs~ directory to another machine to apply the GPG signature. Submitting your signature diff --git a/docker/gitian-bitcoin-host/build-bitcoin.sh b/docker/gitian-bitcoin-host/build-bitcoin.sh index f26e499..7cef303 100755 --- a/docker/gitian-bitcoin-host/build-bitcoin.sh +++ b/docker/gitian-bitcoin-host/build-bitcoin.sh @@ -7,7 +7,9 @@ fi VERSION="$1" -git clone https://github.com/bitcoin/bitcoin.git && \ +if [ ! -d bitcoin ]; then + git clone https://github.com/bitcoin/bitcoin.git || exit $? +fi cd bitcoin && \ git checkout v${VERSION} || exit $? @@ -20,7 +22,7 @@ while read -r URL FNAME; do if [ -z "$URL" ]; then continue fi - wget --no-check-certificate "$URL" -O "$FNAME" || exit $? + wget --continue --no-check-certificate "$URL" -O "$FNAME" || exit $? done < ../../input-sources/${VERSION}.txt || exit $? ## verify that all sources are correct before continuing diff --git a/docker/gitian-host/Dockerfile b/docker/gitian-host/Dockerfile index e0cd10b..0e3ac13 100644 --- a/docker/gitian-host/Dockerfile +++ b/docker/gitian-host/Dockerfile @@ -25,7 +25,7 @@ RUN dpkg-reconfigure locales && dpkg-reconfigure tzdata ## continue installation of gitian-builder packages RUN echo "deb http://http.debian.net/debian wheezy main contrib non-free" > /etc/apt/sources.list.d/non-free.list -RUN apt-get install -y iptables ca-certificates sudo bridge-utils git python-cheetah qemu-utils parted kpartx ruby apt-cacher-ng lxc +RUN apt-get install -y iptables ca-certificates sudo bridge-utils git python-cheetah qemu-utils parted kpartx ruby apt-cacher-ng lxc parallel ## disable password logins, reverse DNS lookups & privilege separation (due to chroot issues on latest docker) RUN sed -i -e 's/^#PasswordAuthentication yes/PasswordAuthentication no/' -e 's/^UsePrivilegeSeparation.*$/UsePrivilegeSeparation no/' /etc/ssh/sshd_config && echo 'UseDNS no' >> /etc/ssh/sshd_config diff --git a/docker/gitian-host/README.md b/docker/gitian-host/README.md index e8a10ed..804c96e 100644 --- a/docker/gitian-host/README.md +++ b/docker/gitian-host/README.md @@ -1,41 +1,51 @@ Gitian host docker container ============================ -This image contains a [Dockerfile](http://docs.docker.io/reference/builder/) to generate a [gitian-builder](https://gitian.org/) host image, that can subsequently be used for reproducible builds using LXC VMs. +The provided [Dockerfile](http://docs.docker.io/reference/builder/) allows to generate a [gitian-builder](https://gitian.org/) host image, that can subsequently be used for reproducible builds using LXC VMs. How this works: -See also https://github.com/devrandom/gitian-builder/issues/53 +Some of the discussions leading to the creation of this set of Dockerfiles/scripts are available on [this issue](https://github.com/devrandom/gitian-builder/issues/53). + +Preamble +-------- + +It is **necessary** that before you using these scripts you read them and understand what they do. +Why? Because your goal is to create a gitian build (deterministic) that has not been tampered with, thus trust shall be correctly attributed during your process. + +See also: +- https://gitian.org/ +- https://en.wikipedia.org/wiki/Web_of_trust +- http://www.dwheeler.com/trusting-trust/ +- https://www.debian.org/ +- https://www.docker.io/ +- http://www.ubuntu.com/ How to build the image ---------------------- -I have not yet pushed images to the [Docker Registry](https://index.docker.io/), but it is a non-issue because you are supposed to create your images from scratch. - -First run **scripts/build-wheezy.sh** to get a Debian Wheezy image debootstrapped from Debian repositories. +Images have not been pushed images to my [Docker Registry](https://index.docker.io/) account, this is on purpose because even if generated images have my repository prefix ('gdm85/') you are supposed +to create them from scratch. **NOTE:** you must have debootstrap on your real host to run this script successfully, and also make sure you have a keyring with APT keys, see also https://wiki.debian.org/SecureApt -At this point run **scripts/create-gitian-host.sh**, this will simply build the Dockerfile that installs the few necessary dependencies inside the prepared image. +First steps: +- run **scripts/build-wheezy.sh** to get a Debian Wheezy image debootstrapped from Debian repositories. +- run **scripts/create-gitian-host.sh**, this will simply build the Dockerfile that installs the few necessary dependencies inside the prepared image, plus generate a second image with the i386 and amd64 VMs (see [build-base-vms.sh](build-base-vms.sh)). + +**NOTE:** when I say "run", what I really mean is "read the script, study it for your own learning purposes, then run it" ;) -Afterwards you can spawn a gitian-host container as follows: +After steps above you will have prepared a full gitian builder environment for deterministic builds. +The image that contains the VMs is called *gdm85/gitian-host-vms*; in future you can spawn containers with this image for new gitian-builder environments. +Example: ``` $ scripts/spawn-gitian-host.sh You can now SSH into container 8a955ff5607b62d4c295745f27bbc38f2e8e011ea93053e641617d50ad2aa5a2: ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@172.17.0.2 $ ``` - -**NOTE:** when I say "run", what I really mean is "read the script, study it for your own learning purposes, then run it" ;) - -This will create a privileged container that you can access with the SSH command displayed. - -First step ----------- - -As first step it is reccomended to run the script ./build-base-vms.sh; this will take a while to create the 2 VMs. -Once done, you have prepared a gitian builder environment for deterministic builds. You might want to stop the container and create an image to store away so that in future you can fork from there for new gitian-builder containers. +This will create a privileged running container that you can access with the SSH command displayed. Derived images -------------- diff --git a/docker/gitian-host/setup.sh b/docker/gitian-host/setup.sh index 0464ec7..be18cc8 100755 --- a/docker/gitian-host/setup.sh +++ b/docker/gitian-host/setup.sh @@ -110,10 +110,13 @@ rmdir /dev/shm ln -s /run/shm /dev/shm ##NOTE: this is setup here instead of Dockerfile because of a Docker glitch -cp /root/authorized_keys /home/debian/.ssh/ && \ -rm /root/authorized_keys && \ -chmod -R go-rwx /home/debian/.ssh && -chown -R debian.debian /home/debian/.ssh || exit $? +AK=/root/authorized_keys +if [ -s $AK ]; then + cp $AK /home/debian/.ssh/ && \ + rm $AK && \ + chmod -R go-rwx /home/debian/.ssh && + chown -R debian.debian /home/debian/.ssh || exit $? +fi ## test that debian user has access to its own .ssh (yes, Docker glitches crawling...) su -c 'cat /home/debian/.ssh/authorized_keys' -l -- debian || exit $? diff --git a/docker/scripts/create-gitian-host.sh b/docker/scripts/create-gitian-host.sh index 80d4041..ccc3417 100755 --- a/docker/scripts/create-gitian-host.sh +++ b/docker/scripts/create-gitian-host.sh @@ -19,6 +19,26 @@ if [ ! -f authorized_keys ]; then fi fi +function wait_for_ssh() { + local IP="$1" + local SECS="$2" + while [ $SECS -gt 0 ]; do + ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP ls >/dev/null 2>/dev/null && return 0 + sleep 1 + let SECS-=1 + done + return 1 +} + +##NOTE: can leave behind a running container of gitian-host docker build --tag=gdm85/gitian-host . && \ -echo "Gitian host image created successfully!" && \ +CID=$(docker run -d --privileged gdm85/gitian-host) && \ +echo "Now building base VMs" && \ +IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) && \ +wait_for_ssh $IP 10 && \ +ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no debian@$IP ./build-base-vms.sh && \ +docker stop $CID && \ +docker commit $CID gdm85/gitian-host-vms && \ +docker rm $CID && \ +echo "Gitian host images created successfully!" && \ echo "You can now spawn containers with spawn-gitian-host.sh" diff --git a/docker/scripts/spawn-gitian-host.sh b/docker/scripts/spawn-gitian-host.sh index 49d8dd3..1418dd9 100755 --- a/docker/scripts/spawn-gitian-host.sh +++ b/docker/scripts/spawn-gitian-host.sh @@ -2,7 +2,7 @@ set -e -CID=$(docker run -d --privileged gdm85/gitian-host) || exit $? +CID=$(docker run -d --privileged gdm85/gitian-host-vms) && \ IP=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' $CID) || exit $? echo "You can now SSH into container $CID:"