Update tools

pkg/chunk/aws_storage_client.go

@@ -255,7 +255,7 @@ func (a awsStorageClient) BatchWrite(ctx context.Context, input WriteBatch) erro
 }
 
 func (a awsStorageClient) QueryPages(ctx context.Context, query IndexQuery, callback func(result ReadBatch, lastPage bool) (shouldContinue bool)) error {
-	sp, ctx := ot.StartSpanFromContext(ctx, "QueryPages", ot.Tag{"tableName", query.TableName}, ot.Tag{"hashValue", query.HashValue})
+	sp, ctx := ot.StartSpanFromContext(ctx, "QueryPages", ot.Tag{Key: "tableName", Value: query.TableName}, ot.Tag{Key: "hashValue", Value: query.HashValue})
 	defer sp.Finish()
 
 	input := &dynamodb.QueryInput{
@@ -555,7 +555,7 @@ var placeholder = []byte{'c'}
 // Structure is identical to BatchWrite(), but operating on different datatypes
 // so cannot share implementation.  If you fix a bug here fix it there too.
 func (a awsStorageClient) getDynamoDBChunks(ctx context.Context, chunks []Chunk) ([]Chunk, error) {
-	sp, ctx := ot.StartSpanFromContext(ctx, "getDynamoDBChunks", ot.Tag{"numChunks", len(chunks)})
+	sp, ctx := ot.StartSpanFromContext(ctx, "getDynamoDBChunks", ot.Tag{Key: "numChunks", Value: len(chunks)})
 	defer sp.Finish()
 	outstanding := dynamoDBReadRequest{}
 	chunksByKey := map[string]Chunk{}

tools/README.md

@@ -2,6 +2,7 @@
 
 Included in this repo are tools shared by weave.git and scope.git.  They include
 
+- ```bazel```: Bazel build rules used in our projects
 - ```build```: a set of docker base-images for building weave
   projects. These should be used instead of giving each project its
   own build image.
@@ -32,7 +33,11 @@ Included in this repo are tools shared by weave.git and scope.git.  They include
 ## Requirements
 
 - ```lint``` requires shfmt to lint sh files; get shfmt with
-  ```go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt```
+```
+curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+chmod +x shfmt
+```
+  (we pin that version, and it doesn't build from the source repo any more)
 
 ## Using build-tools.git
 

tools/bazel/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//proto:compiler.bzl", "go_proto_compiler")
+
+go_proto_compiler(
+    name = "gogo_proto",
+    deps = [
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+    ],
+    plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+    visibility = ["//visibility:public"],
+)
+
+go_proto_compiler(
+    name = "gogo_grpc",
+    deps = [
+        "//vendor/github.com/gogo/protobuf/gogoproto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/proto:go_default_library",
+        "//vendor/github.com/gogo/protobuf/sortkeys:go_default_library",
+        "//vendor/google.golang.org/grpc:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+    ],
+    plugin = "@com_github_gogo_protobuf//protoc-gen-gogoslick",
+    options = ["plugins=grpc"],
+    visibility = ["//visibility:public"],
+)

tools/bazel/gogo.bzl

@@ -0,0 +1,36 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_repository")
+
+
+_BUILD_FILE = """
+proto_library(
+    name = "gogoproto",
+    srcs = ["gogo.proto"],
+    deps = [
+        "@com_google_protobuf//:descriptor_proto",
+    ],
+    visibility = ["//visibility:public"],
+)
+"""
+
+def _go_repository_impl(ctx):
+  ctx.file("BUILD.bazel", content="")
+  ctx.file("github.com/gogo/protobuf/gogoproto/BUILD.bazel", content=_BUILD_FILE)
+  ctx.template("github.com/gogo/protobuf/gogoproto/gogo.proto", ctx.attr._proto)
+
+_gogo_proto_repository = repository_rule(
+    implementation = _go_repository_impl,
+    attrs = {
+        "_proto": attr.label(default="//vendor/github.com/gogo/protobuf/gogoproto:gogo.proto"),
+    },
+)
+
+def gogo_dependencies():
+  go_repository(
+      name = "com_github_gogo_protobuf",
+      importpath = "github.com/gogo/protobuf",
+      urls = ["https://codeload.github.com/ianthehat/protobuf/zip/2adc21fd136931e0388e278825291678e1d98309"],
+      strip_prefix = "protobuf-2adc21fd136931e0388e278825291678e1d98309",
+      type = "zip",
+      build_file_proto_mode="disable",
+  )
+  _gogo_proto_repository(name = "internal_gogo_proto_repository")

tools/build/golang/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.8.0-stretch
+FROM golang:1.9.2-stretch
 RUN apt-get update && \
     apt-get install -y \
       curl \
@@ -11,9 +11,10 @@ RUN apt-get update && \
       python-pip \
       python-requests \
       python-yaml \
+      shellcheck \
       unzip && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install attrs pyhcl
+RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0
 RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
 	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
 	chmod +x shfmt && \

tools/circle.yml

@@ -13,23 +13,13 @@ dependencies:
     - go install -tags netgo std
     - mkdir -p $(dirname $SRCDIR)
     - cp -r $(pwd)/ $SRCDIR
-    - |
-      curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
-        echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
-        chmod +x shfmt && \
-        sudo mv shfmt /usr/bin
-    - |
-      cd $SRCDIR;
-      go get \
-        github.com/fzipp/gocyclo \
-        github.com/golang/lint/golint \
-        github.com/kisielk/errcheck \
-        github.com/fatih/hclfmt
-    - pip install yapf==0.16.2 flake8==3.3.0
 
 test:
   override:
-    - cd $SRCDIR; ./lint .
+    - |
+      cd $SRCDIR/build && \
+      make && \
+      docker run --rm -v "$SRCDIR:$SRCDIR" -w "$SRCDIR" --entrypoint sh quay.io/weaveworks/build-golang -c ./lint .
     - cd $SRCDIR/cover; make
     - cd $SRCDIR/socks; make
     - cd $SRCDIR/runner; make

tools/lint

@@ -6,7 +6,8 @@
 #
 # For shell files, it runs shfmt. If you don't have that installed, you can get
 # it with:
-#   go get -u gopkg.in/mvdan/sh.v1/cmd/shfmt
+#   curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64
+#   chmod +x shfmt
 #
 # With no arguments, it lints the current files staged
 # for git commit.  Or you can pass it explicit filenames
@@ -50,14 +51,6 @@ spell_check() {
     local filename="$1"
     local lint_result=0
 
-    # we don't want to spell check tar balls, binaries, Makefile and json files
-    if file "$filename" | grep executable >/dev/null 2>&1; then
-        return $lint_result
-    fi
-    if [[ $filename == *".tar" || $filename == *".gz" || $filename == *".json" || $(basename "$filename") == "Makefile" ]]; then
-        return $lint_result
-    fi
-
     # misspell is completely optional.  If you don't like it
     # don't have it installed.
     if ! type misspell >/dev/null 2>&1; then
@@ -187,19 +180,24 @@ lint() {
         *.pb.go) return ;;
     esac
 
-    if [[ "$(file --mime-type "${filename}" | awk '{print $2}')" == "text/x-shellscript" ]]; then
-        ext="sh"
-    fi
+    mimetype=$(file --mime-type "${filename}" | awk '{print $2}')
 
-    case "$ext" in
-        go) lint_go "${filename}" || lint_result=1 ;;
-        sh) lint_sh "${filename}" || lint_result=1 ;;
-        tf) lint_tf "${filename}" || lint_result=1 ;;
-        md) lint_md "${filename}" || lint_result=1 ;;
-        py) lint_py "${filename}" || lint_result=1 ;;
+    case "$mimetype.$ext" in
+        text/x-shellscript.*) lint_sh "${filename}" || lint_result=1 ;;
+        *.go) lint_go "${filename}" || lint_result=1 ;;
+        *.tf) lint_tf "${filename}" || lint_result=1 ;;
+        *.md) lint_md "${filename}" || lint_result=1 ;;
+        *.py) lint_py "${filename}" || lint_result=1 ;;
     esac
 
-    spell_check "${filename}" || lint_result=1
+    # we don't want to spell check tar balls, binaries, Makefile and json files
+    case "$mimetype.$ext" in
+        *.tar | *.gz | *.json) ;;
+        *.req | *.key | *.pem | *.crt) ;;
+        application/x-executable.*) ;;
+        text/x-makefile.*) ;;
+        *) spell_check "${filename}" || lint_result=1 ;;
+    esac
 
     return $lint_result
 }
@@ -242,7 +240,7 @@ filter_out() {
 
 list_files() {
     if [ $# -gt 0 ]; then
-        find "$@" | grep -vE '(^|/)vendor/'
+        find "$@" \( -name vendor -o -name .git \) -prune -o -type f
     else
         git ls-files --exclude-standard | grep -vE '(^|/)vendor/'
     fi

tools/provisioning/README.md

@@ -16,16 +16,15 @@ You can then use these machines as is or run various Ansible playbooks from `../
 
   * On macOS: `brew install vagrant`
   * On Linux (via Aptitude): `sudo apt install vagrant`
-  * If you need a specific version:
-
-          curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin
-
   * For other platforms or more details, see [here](https://www.vagrantup.com/docs/installation/)
 
 * You will need [Terraform](https://www.terraform.io) installed on your machine and added to your `PATH` in order to be able to provision cloud-hosted machines automatically.
 
   * On macOS: `brew install terraform`
   * On Linux (via Aptitude): `sudo apt install terraform`
+  * If you need a specific version:
+
+          curl -fsS https://releases.hashicorp.com/terraform/x.y.z/terraform_x.y.z_linux_amd64.zip | gunzip > terraform && chmod +x terraform && sudo mv terraform /usr/bin
   * For other platforms or more details, see [here](https://www.terraform.io/intro/getting-started/install.html)
 
 * Depending on the cloud provider, you may have to create an account, manually onboard, create and register SSH keys, etc. 

tools/push-images

@@ -47,7 +47,7 @@ done
 
 # Wait individually for tasks so we fail-exit on any non-zero return code
 for p in $pids; do
-    wait $p
+    wait "$p"
 done
 
 wait

tools/scheduler/main.py

@@ -107,8 +107,12 @@ def avg(test):
                r'(?P<type>[\w\-]+)$'),
 ]
 NAME_REGEXES = [
-    re.compile(r'^host(?P<index>\d+)-(?P<build>\d+)-(?P<shard>\d+)$'),
-    re.compile(r'^test-(?P<build>\d+)-(?P<shard>\d+)-(?P<index>\d+)$'),
+    re.compile(pat)
+    for pat in (
+        r'^host(?P<index>\d+)-(?P<build>\d+)-(?P<shard>\d+)$',
+        r'^host(?P<index>\d+)-(?P<project>[a-zA-Z0-9-]+)-(?P<build>\d+)'
+        r'-(?P<shard>\d+)$',
+        r'^test-(?P<build>\d+)-(?P<shard>\d+)-(?P<index>\d+)$', )
 ]
 
 

tools/test

@@ -9,13 +9,18 @@ TAGS=
 PARALLEL=
 RACE="-race -covermode=atomic"
 TIMEOUT=1m
+VERBOSE=
 
 usage() {
     echo "$0 [-slow] [-in-container foo] [-netgo] [-(no-)go-get] [-timeout 1m]"
 }
 
 while [ $# -gt 0 ]; do
     case "$1" in
+        "-v")
+            VERBOSE="-v"
+            shift 1
+            ;;
         "-slow")
             SLOW=true
             shift 1
@@ -55,7 +60,7 @@ while [ $# -gt 0 ]; do
     esac
 done
 
-GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT)
+GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT $VERBOSE)
 
 if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then
     SLOW=true