Integration test running ingesters

.circleci/config.yml

@@ -1,4 +1,4 @@
-version: 2
+version: 2.1
 
 # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
 defaults: &defaults
@@ -20,6 +20,8 @@ workflows:
           tags:
             only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
     - integration:
+        requires:
+        - build
         filters:
           tags:
             only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
@@ -39,6 +41,18 @@ workflows:
           tags:
             only: /^v[0-9]+(\.[0-9]+){2}(-.+|[^-.]*)$/
 
+commands:
+  install-docker:
+    steps:
+    - run:
+        name: Install Docker client
+        command: |
+          set -x
+          VER="17.03.0-ce"
+          curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
+          tar -xz -C /tmp -f /tmp/docker-$VER.tgz
+          mv /tmp/docker/* /usr/bin
+
 jobs:
   lint:
     <<: *defaults
@@ -63,33 +77,37 @@ jobs:
         command: make BUILD_IN_CONTAINER=false test
 
   integration:
-    docker:
-    - image: cortexproject/build-image:master-561033f49
-    - image: circleci/postgres:9.6.2-alpine
-      environment:
-        POSTGRES_DB: configs_test
+    machine:
+      image: ubuntu-1604:201903-01
 
-    working_directory: /go/src/github.com/cortexproject/cortex
+    working_directory: ~/src/github.com/cortexproject/cortex
     steps:
     - checkout
+    - restore_cache:
+        key: v1-cortex-{{ .Branch }}-{{ .Revision }}
     - run:
         name: Integration Test
-        command: MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations make BUILD_IN_CONTAINER=false configs-integration-test
+        command: |
+          touch build-image/.uptodate
+          MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations make BUILD_IMAGE=cortexproject/build-image:1370-pin-protoc-7ae297930 configs-integration-test
+    - run:
+        name: Load Images
+        command: ln -s /tmp/images ./images; make BUILD_IN_CONTAINER=false load-images
+    - run:
+        name: Ingester hand-over test
+        command: ./integration-tests/test-handover.sh
+    - run:
+        name: Ingester flush test
+        command: |
+          ./integration-tests/stop-components.sh
+          ./integration-tests/test-flush.sh
 
   build:
     <<: *defaults
     steps:
     - checkout
     - setup_remote_docker
-
-    - run:
-        name: Install Docker client
-        command: |
-          set -x
-          VER="17.03.0-ce"
-          curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
-          tar -xz -C /tmp -f /tmp/docker-$VER.tgz
-          mv /tmp/docker/* /usr/bin
+    - install-docker
 
     - run:
         name: Build
@@ -108,34 +126,29 @@ jobs:
 
     - run:
         name: Save Images
-        command: make BUILD_IN_CONTAINER=false save-images
+        command: |
+          mkdir /tmp/images
+          ln -s /tmp/images ./images
+          make BUILD_IN_CONTAINER=false save-images
 
     - save_cache:
         key: v1-cortex-{{ .Branch }}-{{ .Revision }}
         paths:
-        - images/
+        - /tmp/images/
 
   deploy:
     <<: *defaults
     steps:
     - checkout
     - setup_remote_docker
-
-    - run:
-        name: Install Docker client
-        command: |
-          set -x
-          VER="17.03.0-ce"
-          curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
-          tar -xz -C /tmp -f /tmp/docker-$VER.tgz
-          mv /tmp/docker/* /usr/bin
+    - install-docker
 
     - restore_cache:
         key: v1-cortex-{{ .Branch }}-{{ .Revision }}
 
     - run:
         name: Load Images
-        command: make BUILD_IN_CONTAINER=false load-images
+        command: ln -s /tmp/images ./images; make BUILD_IN_CONTAINER=false load-images
 
     - run:
         name: Deploy

Makefile

@@ -64,6 +64,7 @@ build-image/$(UPTODATE): build-image/*
 # All the boiler plate for building golang follows:
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
+BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image
 # RM is parameterized to allow CircleCI to run builds, as it
 # currently disallows `docker run --rm`. This value is overridden
 # in circle.yml
@@ -93,23 +94,23 @@ exes $(EXES) protos $(PROTO_GOS) lint test shell mod-check check-protos: build-i
 		-v $(shell pwd)/.cache:/go/cache \
 		-v $(shell pwd)/.pkg:/go/pkg \
 		-v $(shell pwd):/go/src/github.com/cortexproject/cortex \
-		$(IMAGE_PREFIX)build-image $@;
+		$(BUILD_IMAGE) $@;
 
 configs-integration-test: build-image/$(UPTODATE)
 	@mkdir -p $(shell pwd)/.pkg
 	@mkdir -p $(shell pwd)/.cache
-	DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=configs_test' postgres:9.6)"; \
-	@echo
-	@echo ">>>> Entering build container: $@"
-	@$(SUDO) docker run $(RM) $(TTY) -i \
+	@DB_CONTAINER="$$(docker run -d -e 'POSTGRES_DB=configs_test' postgres:9.6)"; \
+	echo ; \
+	echo ">>>> Entering build container: $@"; \
+	$(SUDO) docker run $(RM) $(TTY) -i \
 		-v $(shell pwd)/.cache:/go/cache \
 		-v $(shell pwd)/.pkg:/go/pkg \
 		-v $(shell pwd):/go/src/github.com/cortexproject/cortex \
 		-v $(shell pwd)/cmd/cortex/migrations:/migrations \
 		--workdir /go/src/github.com/cortexproject/cortex \
 		--link "$$DB_CONTAINER":configs-db.cortex.local \
 		-e DB_ADDR=configs-db.cortex.local \
-		$(IMAGE_PREFIX)build-image $@; \
+		$(BUILD_IMAGE) $@; \
 	status=$$?; \
 	test -n "$(CIRCLECI)" || docker rm -f "$$DB_CONTAINER"; \
 	exit $$status

integration-tests/common.sh

@@ -0,0 +1,29 @@
+# file intended to be sourced by other shell scripts
+
+TEST_DIR="$( cd "$(dirname "$0")" ; pwd -P )"
+
+IMAGE_TAG=$($TEST_DIR/../tools/image-tag)
+
+COMMON_ARGS="-consul.hostname=consul:8500"
+STORAGE_ARGS="-config-yaml=/tests/schema1.yaml -dynamodb.url=dynamodb://u:[email protected].:8000"
+INGESTER_ARGS="$COMMON_ARGS $STORAGE_ARGS -ingester.num-tokens=4 -ingester.min-ready-duration=1s --ingester.final-sleep=0s -ingester.concurrent-flushes=5"
+RUN_ARGS="--net=cortex -v $TEST_DIR:/tests"
+AVALANCHE_IMAGE=quay.io/freshtracks.io/avalanche:master-2019-07-28-47b00e3
+
+# Execute command $1 repeatedly until it returns true; description in $2, optional repeat limit in $3
+wait_for() {
+    reps=${3:-10}
+    for i in $(seq 1 $reps); do
+        echo "Waiting for $2"
+        if $1; then
+            return
+        fi
+        sleep 1
+    done
+    echo "Timed out waiting for $2" >&2
+    exit 1
+}
+
+container_ip() {
+    docker inspect -f "{{.NetworkSettings.Networks.cortex.IPAddress}}" $@
+}

integration-tests/schema1.yaml

@@ -0,0 +1,10 @@
+configs:
+- from: "2019-03-20"
+  store: aws-dynamo
+  schema: v9
+  index:
+    prefix: cortex_
+    period: 168h0m0s
+  chunks:
+    prefix: cortex_chunks_
+    period: 168h0m0s

integration-tests/start-components.sh

@@ -0,0 +1,12 @@
+#! /bin/sh
+
+set -e
+
+THIS_DIR=$(dirname "$0")
+. "$THIS_DIR/common.sh"
+
+docker network create cortex
+
+docker run $RUN_ARGS -d --name=consul --hostname=consul consul:0.9 agent -ui -server -client=0.0.0.0 -bootstrap
+docker run $RUN_ARGS -d --name=dynamodb --hostname=dynamodb amazon/dynamodb-local:1.11.477 -jar DynamoDBLocal.jar -inMemory -sharedDb
+docker run $RUN_ARGS -d --name=distributor --hostname=distributor -p 8080:80 quay.io/cortexproject/cortex:$IMAGE_TAG -target=distributor $COMMON_ARGS -distributor.replication-factor=1

integration-tests/stop-components.sh

@@ -0,0 +1,6 @@
+#! /bin/sh
+
+set -e
+
+docker rm -f tm i2 i1 distributor dynamodb consul || true
+docker network rm cortex || true

integration-tests/test-flush.sh

@@ -0,0 +1,30 @@
+#! /bin/sh
+
+set -e
+
+. "$(dirname "$0")/common.sh"
+
+"$(dirname "$0")/start-components.sh"
+
+# TODO: wait for DynamoDB ready
+sleep 5
+echo Start table-manager
+docker run $RUN_ARGS -d --name=tm --hostname=tm --restart=always quay.io/cortexproject/cortex:$IMAGE_TAG -target=table-manager $STORAGE_ARGS -table-manager.retention-period=168h
+
+echo Start ingester
+docker run $RUN_ARGS -d --name=i1 --hostname=i1 quay.io/cortexproject/cortex:$IMAGE_TAG -target=ingester $INGESTER_ARGS
+
+sleep 5
+I1_ADDR=$(container_ip i1)
+wait_for "curl -s -f -m 3 $I1_ADDR/ready" "ingester ready"
+
+has_tokens_owned() {
+    curl -s -f -m 3 $1/metrics | grep 'cortex_ring_tokens_owned' >/dev/null
+}
+DIST_ADDR=$(container_ip distributor)
+wait_for "has_tokens_owned $DIST_ADDR" "distributor to see ingester in ring"
+
+docker run $RUN_ARGS --rm $AVALANCHE_IMAGE --metric-count=2 --label-count=2 --series-count=2 --remote-requests-count=1 --remote-url=http://distributor/api/prom/push
+
+echo Stop ingester so it should flush
+docker stop -t=60 i1 # allow 1 minute for flush to complete

integration-tests/test-handover.sh

@@ -0,0 +1,32 @@
+#! /bin/sh
+
+set -e
+
+. "$(dirname "$0")/common.sh"
+
+"$(dirname "$0")/start-components.sh"
+
+# Note we haven't created any tables in the DB yet - we aren't going to run long enough to flush anything
+
+echo Start first ingester
+docker run $RUN_ARGS -d --name=i1 --hostname=i1 quay.io/cortexproject/cortex:$IMAGE_TAG -target=ingester $INGESTER_ARGS -ingester.claim-on-rollout=true
+
+I1_ADDR=$(container_ip i1)
+wait_for "curl -s -f -m 3 $I1_ADDR/ready" "ingester ready"
+
+has_tokens_owned() {
+    curl -s -f -m 3 $1/metrics | grep -q 'cortex_ring_tokens_owned'
+}
+DIST_ADDR=$(container_ip distributor)
+wait_for "has_tokens_owned $DIST_ADDR" "distributor to see ingester in ring"
+
+docker run $RUN_ARGS --rm $AVALANCHE_IMAGE --metric-count=2 --label-count=2 --series-count=2 --remote-requests-count=1 --remote-url=http://distributor/api/prom/push
+
+echo Start second ingester, waiting for first
+docker run $RUN_ARGS -d --name=i2 --hostname=i2 quay.io/cortexproject/cortex:$IMAGE_TAG -target=ingester $INGESTER_ARGS -ingester.join-after=300s -ingester.claim-on-rollout=true
+
+echo Stop first ingester so it should hand over
+docker stop i1
+
+I2_ADDR=$(container_ip i2)
+wait_for "curl -s -f -m 3 $I2_ADDR/ready" "ingester ready"