cortexproject
diff --git a/‎.gitignore
Lines changed: 4 additions & 0 deletions b/‎.gitignore
Lines changed: 4 additions & 0 deletions
diff --git a/‎README.md
Lines changed: 17 additions & 2 deletions b/‎README.md
Lines changed: 17 additions & 2 deletions
diff --git a/‎bazel-rules/BUILD.bazel
Lines changed: 26 additions & 0 deletions b/‎bazel-rules/BUILD.bazel
Lines changed: 26 additions & 0 deletions
diff --git a/‎bazel-rules/gogo.bzl
Lines changed: 36 additions & 0 deletions b/‎bazel-rules/gogo.bzl
Lines changed: 36 additions & 0 deletions
diff --git a/‎build/Makefile
Lines changed: 46 additions & 0 deletions b/‎build/Makefile
Lines changed: 46 additions & 0 deletions
diff --git a/‎build/golang/Dockerfile
Lines changed: 50 additions & 0 deletions b/‎build/golang/Dockerfile
Lines changed: 50 additions & 0 deletions
diff --git a/‎build/golang/build.sh
Lines changed: 22 additions & 0 deletions b/‎build/golang/build.sh
Lines changed: 22 additions & 0 deletions
diff --git a/‎build/haskell/Dockerfile
Lines changed: 4 additions & 0 deletions b/‎build/haskell/Dockerfile
Lines changed: 4 additions & 0 deletions
diff --git a/‎build/haskell/build.sh
Lines changed: 12 additions & 0 deletions b/‎build/haskell/build.sh
Lines changed: 12 additions & 0 deletions
diff --git a/‎build/haskell/copy-libraries
Lines changed: 41 additions & 0 deletions b/‎build/haskell/copy-libraries
Lines changed: 41 additions & 0 deletions
diff --git a/‎circle.yml
Lines changed: 22 additions & 8 deletions b/‎circle.yml
Lines changed: 22 additions & 8 deletions
@@ -4,3 +4,7 @@ socks/image.tar
 runner/runner
 *.pyc
 *~
+terraform.tfstate
+terraform.tfstate.backup
+*.retry
+build/**/.uptodate
@@ -2,12 +2,18 @@
 
 Included in this repo are tools shared by weave.git and scope.git.  They include
 
+- ```bazel-rules```: Bazel build rules used in our projects
+- ```build```: a set of docker base-images for building weave
+  projects. These should be used instead of giving each project its
+  own build image.
+- ```provisioning```: a set of Terraform scripts to provision virtual machines in GCP, AWS or Digital Ocean.
+- ```config_management```: a set of Ansible playbooks to configure virtual machines for development, testing, etc.
 - ```cover```: a tool which merges overlapping coverage reports generated by go
   test
 - ```files-with-type```: a tool to search directories for files of a given
   MIME type
-- ```lint```: a script to lint Go project; runs various tools like golint, go
-  vet, errcheck etc
+- ```lint```: a script to lint go, sh and hcl files; runs various tools like
+  golint, go vet, errcheck, shellcheck etc
 - ```rebuild-image```: a script to rebuild docker images when their input files
   change; useful when you using docker images to build your software, but you
   don't want to build the image every time.
@@ -24,6 +30,15 @@ Included in this repo are tools shared by weave.git and scope.git.  They include
 - ```scheduler```: an appengine application that can be used to distribute
   tests across different shards in CircleCI.
 
+## Requirements
+
+- ```lint``` requires shfmt to lint sh files; get shfmt with
+```
+curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+chmod +x shfmt
+```
+  (we pin that version, and it doesn't build from the source repo any more)
+
 ## Using build-tools.git
 
 To allow you to tie your code to a specific version of build-tools.git, such
 
@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+
+go_proto_compiler(
+    name = "gogo_proto",
+    deps = [
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+    ],
+    plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+    visibility = ["//visibility:public"],
+)
+
+go_proto_compiler(
+    name = "gogo_grpc",
+    deps = [
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+    ],
+    plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+    options = ["plugins=grpc"],
+    visibility = ["//visibility:public"],
+)
@@ -0,0 +1,36 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_repository")
+
+
+_BUILD_FILE = """
+proto_library(
+    name = "gogoproto",
+    srcs = ["gogo.proto"],
+    deps = [
+        "@com_google_protobuf//:descriptor_proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+"""
+
+def _go_repository_impl(ctx):
+  ctx.file("BUILD.bazel", content="")
+  ctx.file("github.com/gogo/protobuf/gogoproto/BUILD.bazel", content=_BUILD_FILE)
+  ctx.template("github.com/gogo/protobuf/gogoproto/gogo.proto", ctx.attr._proto)
+
+_gogo_proto_repository = repository_rule(
+    implementation = _go_repository_impl,
+    attrs = {
+        "_proto": attr.label(default="//vendor/github.com/gogo/protobuf/gogoproto:gogo.proto"),
+    },
+)
+
+def gogo_dependencies():
+  go_repository(
+      name = "com_github_gogo_protobuf",
+      importpath = "github.com/gogo/protobuf",
+      urls = ["https://codeload.github.com/ianthehat/protobuf/zip/2adc21fd136931e0388e278825291678e1d98309"],
+      strip_prefix = "protobuf-2adc21fd136931e0388e278825291678e1d98309",
+      type = "zip",
+      build_file_proto_mode="disable",
+  )
+  _gogo_proto_repository(name = "internal_gogo_proto_repository")
@@ -0,0 +1,46 @@
+.PHONY: all clean images
+.DEFAULT_GOAL := all
+
+# Boiler plate for bulding Docker containers.
+# All this must go at top of file I'm afraid.
+IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_TAG := $(shell ../image-tag)
+UPTODATE := .uptodate
+
+# Every directory with a Dockerfile in it builds an image called
+# $(IMAGE_PREFIX)<dirname>. Dependencies (i.e. things that go in the image)
+# still need to be explicitly declared.
+%/$(UPTODATE): %/Dockerfile %/*
+	$(SUDO) docker build -t $(IMAGE_PREFIX)$(shell basename $(@D)) $(@D)/
+	$(SUDO) docker tag $(IMAGE_PREFIX)$(shell basename $(@D)) $(IMAGE_PREFIX)$(shell basename $(@D)):$(IMAGE_TAG)
+	touch $@
+
+# Get a list of directories containing Dockerfiles
+DOCKERFILES := $(shell find . -name tools -prune -o -name vendor -prune -o -type f -name 'Dockerfile' -print)
+UPTODATE_FILES := $(patsubst %/Dockerfile,%/$(UPTODATE),$(DOCKERFILES))
+DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES))
+IMAGE_NAMES := $(foreach dir,$(DOCKER_IMAGE_DIRS),$(patsubst %,$(IMAGE_PREFIX)%,$(shell basename $(dir))))
+images:
+	$(info $(IMAGE_NAMES))
+	@echo > /dev/null
+
+# Define imagetag-golang, etc, for each image, which parses the dockerfile and
+# prints an image tag. For example:
+#     FROM golang:1.8.1-stretch
+# in the "foo/Dockerfile" becomes:
+#     $ make imagetag-foo
+#     1.8.1-stretch
+define imagetag_dep
+.PHONY: imagetag-$(1)
+$(patsubst $(IMAGE_PREFIX)%,imagetag-%,$(1)): $(patsubst $(IMAGE_PREFIX)%,%,$(1))/Dockerfile
+	@cat $$< | grep "^FROM " | head -n1 | sed 's/FROM \(.*\):\(.*\)/\2/'
+endef
+$(foreach image, $(IMAGE_NAMES), $(eval $(call imagetag_dep, $(image))))
+
+all: $(UPTODATE_FILES)
+
+clean:
+	$(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true
+	rm -rf $(UPTODATE_FILES)
+
+
@@ -0,0 +1,50 @@
+FROM golang:1.9.2-stretch
+RUN apt-get update && \
+    apt-get install -y \
+      curl \
+      file \
+      git \
+      jq \
+      libprotobuf-dev \
+      make \
+      protobuf-compiler \
+      python-pip \
+      python-requests \
+      python-yaml \
+      shellcheck \
+      unzip && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0
+RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
+	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
+	chmod +x shfmt && \
+	mv shfmt /usr/bin
+RUN go clean -i net && \
+	go install -tags netgo std && \
+	go install -race -tags netgo std
+RUN go get -tags netgo \
+		github.com/FiloSottile/gvt \
+		github.com/client9/misspell/cmd/misspell \
+		github.com/fatih/hclfmt \
+		github.com/fzipp/gocyclo \
+		github.com/gogo/protobuf/gogoproto \
+		github.com/gogo/protobuf/protoc-gen-gogoslick \
+		github.com/golang/dep/... \
+		github.com/golang/lint/golint \
+		github.com/golang/protobuf/protoc-gen-go \
+		github.com/kisielk/errcheck \
+		github.com/mjibson/esc \
+		github.com/prometheus/prometheus/cmd/promtool && \
+		rm -rf /go/pkg /go/src
+RUN mkdir protoc && \
+	cd protoc && \
+	curl -O -L https://github.com/google/protobuf/releases/download/v3.1.0/protoc-3.1.0-linux-x86_64.zip && \
+	unzip protoc-3.1.0-linux-x86_64.zip && \
+	cp bin/protoc /usr/bin/ && \
+	chmod o+x /usr/bin/protoc && \
+	cd .. && \
+	rm -rf protoc
+RUN mkdir -p /var/run/secrets/kubernetes.io/serviceaccount && \
+		touch /var/run/secrets/kubernetes.io/serviceaccount/token
+COPY build.sh /
+ENTRYPOINT ["/build.sh"]
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -eu
+
+if [ -n "${SRC_NAME:-}" ]; then
+    SRC_PATH=${SRC_PATH:-$GOPATH/src/$SRC_NAME}
+elif [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set either \$SRC_NAME or \$SRC_PATH."
+    exit 1
+fi
+
+# If we run make directly, any files created on the bind mount
+# will have awkward ownership.  So we switch to a user with the
+# same user and group IDs as source directory.  We have to set a
+# few things up so that sudo works without complaining later on.
+uid=$(stat --format="%u" "$SRC_PATH")
+gid=$(stat --format="%g" "$SRC_PATH")
+echo "weave:x:$uid:$gid::$SRC_PATH:/bin/sh" >>/etc/passwd
+echo "weave:*:::::::" >>/etc/shadow
+echo "weave	ALL=(ALL)	NOPASSWD: ALL" >>/etc/sudoers
+
+su weave -c "PATH=$PATH make -C $SRC_PATH BUILD_IN_CONTAINER=false $*"
@@ -0,0 +1,4 @@
+FROM fpco/stack-build:lts-8.9
+COPY build.sh /
+COPY copy-libraries /usr/local/bin/
+ENTRYPOINT ["/build.sh"]
@@ -0,0 +1,12 @@
+#!/bin/sh
+#
+# Build a static Haskell binary using stack.
+
+set -eu
+
+if [ -z "${SRC_PATH:-}" ]; then
+    echo "Must set \$SRC_PATH."
+    exit 1
+fi
+
+make -C "$SRC_PATH" BUILD_IN_CONTAINER=false "$@"
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Copy dynamically linked libraries for a binary, so we can assemble a Docker
+# image.
+#
+# Run with:
+#   copy-libraries /path/to/binary /output/dir
+#
+# Dependencies:
+# - awk
+# - cp
+# - grep
+# - ldd
+# - mkdir
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# Path to a Linux binary that we're going to run in the container.
+binary_path="${1}"
+# Path to directory to write the output to.
+output_dir="${2}"
+
+exe_name=$(basename "${binary_path}")
+
+# Identify linked libraries.
+libraries=($(ldd "${binary_path}" | awk '{print $(NF-1)}' | grep -v '=>'))
+# Add /bin/sh, which we need for Docker imports.
+libraries+=('/bin/sh')
+
+mkdir -p "${output_dir}"
+
+# Copy executable and all needed libraries into temporary directory.
+cp "${binary_path}" "${output_dir}/${exe_name}"
+for lib in "${libraries[@]}"; do
+    mkdir -p "${output_dir}/$(dirname "$lib")"
+    # Need -L to make sure we get actual libraries & binaries, not symlinks to
+    # them.
+    cp -L "${lib}" "${output_dir}/${lib}"
+done
@@ -13,18 +13,32 @@ dependencies:
     - go install -tags netgo std
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
-    - |
-      cd $SRCDIR;
-      go get \
-        github.com/fzipp/gocyclo \
-        github.com/golang/lint/golint \
-        github.com/kisielk/errcheck \
-        gopkg.in/mvdan/sh.v1/cmd/shfmt
 
 test:
   override:
-    - cd $SRCDIR; ./lint .
+    - |
+      cd $SRCDIR/build && \
+      make && \
+      docker run --rm -v "$SRCDIR:$SRCDIR" -w "$SRCDIR" --entrypoint sh quay.io/weaveworks/build-golang -c ./lint .
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make
 
+deployment:
+  snapshot:
+    branch: master
+    commands:
+      - docker login -e "$DOCKER_REGISTRY_EMAIL" -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASS" "$DOCKER_REGISTRY_URL"
+      - |
+        cd $SRCDIR/build;
+        for image in $(make images); do
+          # Tag the built images with the revision of this repo.
+          docker push "${image}:${GIT_TAG}"
+
+          # Tag the built images with something derived from the base images in
+          # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+          # base image would lead to a tag of "1.8.0-stretch"
+          IMG_TAG=$(make "imagetag-${image#quay.io/weaveworks/build-}")
+          docker tag "${image}:latest" "${image}:${IMG_TAG}"
+          docker push "${image}:${IMG_TAG}"
+        done