Port Cortex to use Prometheus 2 packages (#583)

* Vendor update for Prometheus 2 port

* Hard-fork still-required Prometheus 1 storage packages

* Update build image to latest Go version

* Add lint/vat exceptions

The updated `go vet` complains about more errors than the old one.

* Re-add flag for configuring log level ourselves

The upstream prometheus/common/log changed and doesn't automatically
register flags anymore.

* Adjust multitenant Alertmanager to new upstream packages

* Implement new querier wrappers for Prom 2 packages

* Other adjustments to new packages and types

* Remove now-obsolete MergeSeriesIterator

* Remove now-obsolete LazySeriesIterator

The laziness was actually never used.

* Wire up new API, querier, etc. in main functions

* Update/fix `go test` -tags separation syntax

* Remove unused metadataSeriesSet type

* Re-add Bazel build files

* Adjust comment about (now-removed) lazy iterators

* Pass correct metric name into getMetricNameMatrix()

* Pass in go-kit logger to alerting rule creation

* Fix ErrNoMetricNameNotSupported message

Only the v8 schema supports omitting the metric name in a query.

* Expand and improve chunk store tests and fix uncovered bugs

- Fix over-selection bug when the metric name was non-equals-matched and
  one selects a series that has a subset of labels of another.
- Regex matches against the metric name were broken because the metric
  name was equals-compared to the matcher regex instead of applying it as
  a regex.
- Unneeded sorting of label matchers has been removed.

* Add back -log.level flag to ingester as well

* Ensure sorting of series labels during creation

* Add test for querier label sorting

* Remove SchemaConfig dependency from ingester

The ingester only needs the MaxChunkAge from the SchemaConfig, and the
ingester.Config is actually a better place to store and configure that
information authoritatively.

* Resolve build problems after master rebase

* Reparse AM fallback config every time so we can mutate it

The deep copy that dumped it as JSON and loaded it again stumbled over
the fact that the JSON marshaling renders Secret fields as <secret> and
thus loses the original secret field contents when reloading it from
JSON.

* Fix flaky chunk test

Author: juliusv

Gopkg.lock

@@ -4,7 +4,11 @@
 
 [[constraint]]
   name = "github.com/prometheus/alertmanager"
-  revision = "0ecc59076ca6b4cbb63252fa7720a3d89d1c81d3"
+  revision = "4a8e71069177280d0317ccd80a8201a06f2d8139"
+
+[[constraint]]
+  name = "github.com/prometheus/prometheus"
+  branch = "master"
 
 [[constraint]]
   name = "github.com/weaveworks/common"
@@ -14,13 +18,21 @@
   name = "github.com/weaveworks/promrus"
   version = "v1.1.0-legacy"
 
+[[override]]
+  name = "k8s.io/client-go"
+  revision = "3627aeb7d4f6ade38f995d2c923e459146493c7e"
+
+[[override]]
+  name = "k8s.io/apimachinery"
+  revision = "75b8dd260ef0469d96d578705a87cffd0e09dab8"
+
 [[override]]
   name = "github.com/Azure/azure-sdk-for-go"
   revision = "bd73d950fa4440dae889bd9917bff7cef539f86e"
 
 [[override]]
   name = "github.com/prometheus/common"
-  revision = "49fee292b27bfff7f354ee0f64e1bc4850462edf"
+  revision = "1bab55dd05dbff384524a6a1c99006d9eb5f139b"
 
 [[override]]
   name = "github.com/weaveworks/mesh"

Gopkg.toml

@@ -106,7 +106,7 @@ shell: build-image/$(UPTODATE)
 	bash
 
 configs-integration-test:
-	/bin/bash -c "go test -tags netgo,integration -timeout 30s ./pkg/configs/..."
+	/bin/bash -c "go test -tags 'netgo integration' -timeout 30s ./pkg/configs/..."
 
 endif
 

Makefile

@@ -1,4 +1,4 @@
-FROM golang:1.8.3-stretch
+FROM golang:1.9.1-stretch
 RUN apt-get update && apt-get install -y python-requests python-yaml file jq unzip protobuf-compiler libprotobuf-dev && \
 	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 RUN go clean -i net && \

build-image/Dockerfile

@@ -8,8 +8,10 @@ go_library(
     deps = [
         "//pkg/alertmanager:go_default_library",
         "//pkg/util:go_default_library",
+        "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/alertmanager/BUILD.bazel

@@ -36,7 +36,7 @@ func main() {
 		}
 		alertmanagerConfig alertmanager.MultitenantAlertmanagerConfig
 	)
-	util.RegisterFlags(&serverConfig, &alertmanagerConfig)
+	util.RegisterFlags(&serverConfig, &alertmanagerConfig, util.LogLevel{})
 	flag.Parse()
 
 	hook := promrus.MustNewPrometheusHook()

cmd/alertmanager/main.go

@@ -12,6 +12,7 @@ go_library(
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/configs/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/distributor/BUILD.bazel

@@ -44,7 +44,7 @@ func main() {
 		ringConfig        ring.Config
 		distributorConfig distributor.Config
 	)
-	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig)
+	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, util.LogLevel{})
 	flag.Parse()
 
 	log.AddHook(promrus.MustNewPrometheusHook())

cmd/distributor/main.go

@@ -15,6 +15,7 @@ go_library(
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/ingester/BUILD.bazel

@@ -34,8 +34,9 @@ func main() {
 	// Ingester needs to know our gRPC listen port.
 	ingesterConfig.ListenPort = &serverConfig.GRPCListenPort
 	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &storageConfig,
-		&schemaConfig, &ingesterConfig)
+		&schemaConfig, &ingesterConfig, util.LogLevel{})
 	flag.Parse()
+	schemaConfig.MaxChunkAge = ingesterConfig.MaxChunkAge
 
 	log.AddHook(promrus.MustNewPrometheusHook())
 
@@ -56,7 +57,7 @@ func main() {
 	}
 	defer chunkStore.Stop()
 
-	ingester, err := ingester.New(ingesterConfig, schemaConfig, chunkStore)
+	ingester, err := ingester.New(ingesterConfig, chunkStore)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/ingester/main.go

@@ -18,12 +18,13 @@ go_library(
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/prometheus/common/route:go_default_library",
+        "//vendor/github.com/prometheus/prometheus/config:go_default_library",
         "//vendor/github.com/prometheus/prometheus/promql:go_default_library",
         "//vendor/github.com/prometheus/prometheus/web/api/v1:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
         "//vendor/github.com/weaveworks/common/user:go_default_library",
-        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/lite/BUILD.bazel

@@ -7,9 +7,9 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
 	"github.com/prometheus/common/route"
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/web/api/v1"
-	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
 	"github.com/weaveworks/common/middleware"
@@ -48,9 +48,10 @@ func main() {
 	// Ingester needs to know our gRPC listen port.
 	ingesterConfig.ListenPort = &serverConfig.GRPCListenPort
 	util.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig,
-		&ingesterConfig, &rulerConfig, &storageConfig, &schemaConfig)
+		&ingesterConfig, &rulerConfig, &storageConfig, &schemaConfig, util.LogLevel{})
 	flag.BoolVar(&unauthenticated, "unauthenticated", false, "Set to true to disable multitenancy.")
 	flag.Parse()
+	schemaConfig.MaxChunkAge = ingesterConfig.MaxChunkAge
 
 	log.AddHook(promrus.MustNewPrometheusHook())
 
@@ -85,7 +86,7 @@ func main() {
 	defer dist.Stop()
 	prometheus.MustRegister(dist)
 
-	ingester, err := ingester.New(ingesterConfig, schemaConfig, chunkStore)
+	ingester, err := ingester.New(ingesterConfig, chunkStore)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -118,13 +119,19 @@ func main() {
 		defer rulerServer.Stop()
 	}
 
-	queryable := querier.NewQueryable(dist, chunkStore)
-	engine := promql.NewEngine(queryable, nil)
-	api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable},
-		querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{})
-	promRouter := route.New(func(r *http.Request) (context.Context, error) {
-		return r.Context(), nil
-	}).WithPrefix("/api/prom/api/v1")
+	sampleQueryable := querier.NewQueryable(dist, chunkStore, false)
+	metadataQueryable := querier.NewQueryable(dist, chunkStore, true)
+
+	engine := promql.NewEngine(sampleQueryable, nil)
+	api := v1.NewAPI(
+		engine,
+		metadataQueryable,
+		querier.DummyTargetRetriever{},
+		querier.DummyAlertmanagerRetriever{},
+		func() config.Config { return config.Config{} },
+		func(f http.HandlerFunc) http.HandlerFunc { return f },
+	)
+	promRouter := route.New().WithPrefix("/api/prom/api/v1")
 	api.Register(promRouter)
 
 	activeMiddleware := middleware.AuthenticateUser
@@ -139,7 +146,7 @@ func main() {
 
 	subrouter := server.HTTP.PathPrefix("/api/prom").Subrouter()
 	subrouter.PathPrefix("/api/v1").Handler(activeMiddleware.Wrap(promRouter))
-	subrouter.Path("/read").Handler(activeMiddleware.Wrap(http.HandlerFunc(queryable.Q.RemoteReadHandler)))
+	subrouter.Path("/read").Handler(activeMiddleware.Wrap(http.HandlerFunc(sampleQueryable.RemoteReadHandler)))
 	subrouter.Path("/validate_expr").Handler(activeMiddleware.Wrap(http.HandlerFunc(dist.ValidateExprHandler)))
 	subrouter.Path("/user_stats").Handler(activeMiddleware.Wrap(http.HandlerFunc(dist.UserStatsHandler)))
 

cmd/lite/main.go

@@ -15,11 +15,12 @@ go_library(
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/prometheus/common/route:go_default_library",
+        "//vendor/github.com/prometheus/prometheus/config:go_default_library",
         "//vendor/github.com/prometheus/prometheus/promql:go_default_library",
         "//vendor/github.com/prometheus/prometheus/web/api/v1:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
-        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/querier/BUILD.bazel

@@ -4,12 +4,12 @@ import (
 	"flag"
 	"net/http"
 
-	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
 	"github.com/prometheus/common/route"
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/web/api/v1"
 
@@ -39,7 +39,7 @@ func main() {
 		storageConfig     storage.Config
 	)
 	util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig,
-		&chunkStoreConfig, &schemaConfig, &storageConfig)
+		&chunkStoreConfig, &schemaConfig, &storageConfig, util.LogLevel{})
 	flag.Parse()
 
 	log.AddHook(promrus.MustNewPrometheusHook())
@@ -75,18 +75,24 @@ func main() {
 	}
 	defer chunkStore.Stop()
 
-	queryable := querier.NewQueryable(dist, chunkStore)
-	engine := promql.NewEngine(queryable, nil)
-	api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable},
-		querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{})
-	promRouter := route.New(func(r *http.Request) (context.Context, error) {
-		return r.Context(), nil
-	}).WithPrefix("/api/prom/api/v1")
+	sampleQueryable := querier.NewQueryable(dist, chunkStore, false)
+	metadataQueryable := querier.NewQueryable(dist, chunkStore, true)
+
+	engine := promql.NewEngine(sampleQueryable, nil)
+	api := v1.NewAPI(
+		engine,
+		metadataQueryable,
+		querier.DummyTargetRetriever{},
+		querier.DummyAlertmanagerRetriever{},
+		func() config.Config { return config.Config{} },
+		func(f http.HandlerFunc) http.HandlerFunc { return f },
+	)
+	promRouter := route.New().WithPrefix("/api/prom/api/v1")
 	api.Register(promRouter)
 
 	subrouter := server.HTTP.PathPrefix("/api/prom").Subrouter()
 	subrouter.PathPrefix("/api/v1").Handler(middleware.AuthenticateUser.Wrap(promRouter))
-	subrouter.Path("/read").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(queryable.Q.RemoteReadHandler)))
+	subrouter.Path("/read").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(sampleQueryable.RemoteReadHandler)))
 	subrouter.Path("/validate_expr").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(dist.ValidateExprHandler)))
 	subrouter.Path("/user_stats").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(dist.UserStatsHandler)))
 

cmd/querier/main.go

@@ -16,6 +16,7 @@ go_library(
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/ruler/BUILD.bazel

@@ -12,6 +12,7 @@ go_library(
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/weaveworks/common/middleware:go_default_library",
         "//vendor/github.com/weaveworks/common/server:go_default_library",
+        "//vendor/github.com/weaveworks/promrus:go_default_library",
         "//vendor/google.golang.org/grpc:go_default_library",
     ],
 )

cmd/table-manager/BUILD.bazel

@@ -28,9 +28,9 @@ go_library(
         "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/prometheus/common/log:go_default_library",
         "//vendor/github.com/prometheus/common/route:go_default_library",
+        "//vendor/github.com/prometheus/prometheus/pkg/labels:go_default_library",
         "//vendor/github.com/weaveworks/common/instrument:go_default_library",
         "//vendor/github.com/weaveworks/common/user:go_default_library",
         "//vendor/github.com/weaveworks/mesh:go_default_library",
-        "//vendor/golang.org/x/net/context:go_default_library",
     ],
 )

pkg/alertmanager/BUILD.bazel

@@ -23,6 +23,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
 	"github.com/prometheus/common/route"
+	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/weaveworks/mesh"
 )
 
@@ -108,16 +109,23 @@ func New(cfg *Config) (*Alertmanager, error) {
 		am.wg.Done()
 	}()
 
-	am.alerts, err = mem.NewAlerts(cfg.DataDir)
+	marker := types.NewMarker()
+	am.alerts, err = mem.NewAlerts(marker, 30*time.Minute, cfg.DataDir)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create alerts: %v", err)
 	}
 
-	am.api = api.New(am.alerts, am.silences, func() dispatch.AlertOverview {
-		return am.dispatcher.Groups()
-	})
+	am.api = api.New(
+		am.alerts,
+		am.silences,
+		func(matchers []*labels.Matcher) dispatch.AlertOverview {
+			return am.dispatcher.Groups(matchers)
+		},
+		marker.Status,
+		nil, // Passing a nil mesh router since we don't show mesh router information in Cortex anyway.
+	)
 
-	am.router = route.New(nil)
+	am.router = route.New()
 
 	webReload := make(chan struct{})
 	ui.Register(am.router.WithPrefix(am.cfg.ExternalURL.Path), webReload)
@@ -157,7 +165,7 @@ func (am *Alertmanager) ApplyConfig(conf *config.Config) error {
 	}
 	tmpl.ExternalURL = am.cfg.ExternalURL
 
-	err = am.api.Update(conf.String(), time.Duration(conf.Global.ResolveTimeout))
+	err = am.api.Update(conf, time.Duration(conf.Global.ResolveTimeout))
 	if err != nil {
 		return err
 	}