Update build tools to 35679ee

tools/.gitignore

@@ -4,3 +4,7 @@ socks/image.tar
 runner/runner
 *.pyc
 *~
+terraform.tfstate
+terraform.tfstate.backup
+*.retry
+build/**/.uptodate

tools/README.md

@@ -2,12 +2,17 @@
 
 Included in this repo are tools shared by weave.git and scope.git.  They include
 
+- ```build```: a set of docker base-images for building weave
+  projects. These should be used instead of giving each project its
+  own build image.
+- ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean.
+- ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc.
 - ```cover```: a tool which merges overlapping coverage reports generated by go
   test
 - ```files-with-type```: a tool to search directories for files of a given
   MIME type
-- ```lint```: a script to lint Go project; runs various tools like golint, go
-  vet, errcheck etc
+- ```lint```: a script to lint go, sh and hcl files; runs various tools like
+  golint, go vet, errcheck, shellcheck etc
 - ```rebuild-image```: a script to rebuild docker images when their input files
   change; useful when you using docker images to build your software, but you
   don't want to build the image every time.
@@ -24,6 +29,11 @@ Included in this repo are tools shared by weave.git and scope.git.  They include
 - ```scheduler```: an appengine application that can be used to distribute
   tests across different shards in CircleCI.
 
+## Requirements
+
+- ```lint``` requires shfmt to lint sh files; get shfmt with
+  ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt```
+
 ## Using build-tools.git
 
 To allow you to tie your code to a specific version of build-tools.git, such

tools/build/Makefile

@@ -0,0 +1,46 @@
+.PHONY: all clean images
+.DEFAULT_GOAL := all
+
+# Boiler plate for bulding Docker containers.
+# All this must go at top of file I'm afraid.
+IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_TAG := $(shell ../image-tag)
+UPTODATE := .uptodate
+
+# Every directory with a Dockerfile in it builds an image called
+# $(IMAGE_PREFIX)<dirname>. Dependencies (i.e. things that go in the image)
+# still need to be explicitly declared.
+%/$(UPTODATE): %/Dockerfile %/*
+	$(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+	$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
+	touch $@
+
+# Get a list of directories containing Dockerfiles
+DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print)
+UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES))
+DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
+IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))))
+images:
+	$(info $(IMAGE_NAMES))
+	@echo > /dev/null
+
+# Define imagetag-golang, etc, for each image, which parses the dockerfile and
+# prints an image tag. For example:
+#     FROM golang:1.8.1-stretch
+# in the "foo/Dockerfile" becomes:
+#     $ make imagetag-foo
+#     1.8.1-stretch
+define imagetag_dep
+.PHONY: imagetag-$(1)
+$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile
+	@cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/'
+endef
+$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image))))
+
+all: $(UPTODATE_FILES)
+
+clean:
+	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
+	rm -rf $(UPTODATE_FILES)
+
+

tools/build/golang/Dockerfile

@@ -0,0 +1,49 @@
+FROM golang:1.8.0-stretch
+RUN apt-get update && \
+    apt-get install -y \
+      curl \
+      file \
+      git \
+      jq \
+      libprotobuf-dev \
+      make \
+      protobuf-compiler \
+      python-pip \
+      python-requests \
+      python-yaml \
+      unzip && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install attrs pyhcl
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+	chmod +x shfmt && \
+	mv shfmt /usr/bin
+RUN go clean -i net && \
+	go install -tags netgo std && \
+	go install -race -tags netgo std
+RUN go get -tags netgo \
+		github.com/FiloSottile/gvt \
+		github.com/client9/misspell/cmd/misspell \
+		github.com/fatih/hclfmt \
+		github.com/fzipp/gocyclo \
+		github.com/gogo/protobuf/gogoproto \
+		github.com/gogo/protobuf/protoc-gen-gogoslick \
+		github.com/golang/dep/... \
+		github.com/golang/lint/golint \
+		github.com/golang/protobuf/protoc-gen-go \
+		github.com/kisielk/errcheck \
+		github.com/mjibson/esc \
+		github.com/prometheus/prometheus/cmd/promtool && \
+		rm -rf /go/pkg /go/src
+RUN mkdir protoc && \
+	cd protoc && \
+	curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \
+	unzip protoc-3.1.0-linux-x86_64.zip && \
+	cp bin/protoc /usr/bin/ && \
+	chmod o+x /usr/bin/protoc && \
+	cd .. && \
+	rm -rf protoc
+RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+		touch /var/run/secrets/kubernetes.io/serviceaccount/token
+COPY build.sh /
+ENTRYPOINT ["/build.sh"]

tools/build/golang/build.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+if [ -n "${SRC_NAME:-}" ]; then
+    SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME}
+elif [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set either \$SRC_NAME or \$SRC_PATH."
+    exit 1
+fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
+echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*"

tools/build/haskell/Dockerfile

@@ -0,0 +1,4 @@
+FROM fpco/stack-build:lts-8.9
+COPY build.sh /
+COPY copy-libraries /usr/local/bin/
+ENTRYPOINT ["/build.sh"]

tools/build/haskell/build.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# Build a static Haskell binary using stack.
+
+set -eu
+
+if [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set \$SRC_PATH."
+    exit 1
+fi
+
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"

tools/build/haskell/copy-libraries

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copy dynamically linked libraries for a binary, so we can assemble a Docker
+# image.
+#
+# Run with:
+#   copy-libraries /path/to/binary /output/dir
+#
+# Dependencies:
+# - awk
+# - cp
+# - grep
+# - ldd
+# - mkdir
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Path to a Linux binary that we're going to run in the container.
+binary_path="${1}"
+# Path to directory to write the output to.
+output_dir="${2}"
+
+exe_name=$(basename "${binary_path}")
+
+# Identify linked libraries.
+libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>'))
+# Add /bin/sh, which we need for Docker imports.
+libraries+=('/bin/sh')
+
+mkdir -p "${output_dir}"
+
+# Copy executable and all needed libraries into temporary directory.
+cp "${binary_path}" "${output_dir}/${exe_name}"
+for lib in "${libraries[@]}"; do
+    mkdir -p "${output_dir}/$(dirname "$lib")"
+    # Need -L to make sure we get actual libraries & binaries, not symlinks to
+    # them.
+    cp -L "${lib}" "${output_dir}/${lib}"
+done

tools/circle.yml

@@ -13,18 +13,43 @@ dependencies:
     - go install -tags netgo std
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
+    - |
+      curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+        echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+        chmod +x shfmt && \
+        sudo mv shfmt /usr/bin
     - |
       cd $SRCDIR;
       go get \
         github.com/fzipp/gocyclo \
         github.com/golang/lint/golint \
         github.com/kisielk/errcheck \
-        gopkg.in/mvdan/sh.v1/cmd/shfmt
+        github.com/fatih/hclfmt
+    - pip install yapf==0.16.2 flake8==3.3.0
 
 test:
   override:
     - cd $SRCDIR; ./lint .
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
+    - cd $SRCDIR/build; make
+
+deployment:
+  snapshot:
+    branch: master
+    commands:
+      - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL"
+      - |
+        cd $SRCDIR/build;
+        for image in $(make images); do
+          # Tag the built images with the revision of this repo.
+          docker push "${image}:${GIT_TAG}"
 
+          # Tag the built images with something derived from the base images in
+          # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+          # base image would lead to a tag of "1.8.0-stretch"
+          IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}")
+          docker tag "${image}:latest" "${image}:${IMG_TAG}"
+          docker push "${image}:${IMG_TAG}"
+        done