Skip to content

Commit d714b62

Browse files
juliusvjuliusv
authored
Port Cortex to use Prometheus 2 packages (#583)
* Vendor update for Prometheus 2 port * Hard-fork still-required Prometheus 1 storage packages * Update build image to latest Go version * Add lint/vat exceptions The updated `go vet` complains about more errors than the old one. * Re-add flag for configuring log level ourselves The upstream prometheus/common/log changed and doesn't automatically register flags anymore. * Adjust multitenant Alertmanager to new upstream packages * Implement new querier wrappers for Prom 2 packages * Other adjustments to new packages and types * Remove now-obsolete MergeSeriesIterator * Remove now-obsolete LazySeriesIterator The laziness was actually never used. * Wire up new API, querier, etc. in main functions * Update/fix `go test` -tags separation syntax * Remove unused metadataSeriesSet type * Re-add Bazel build files * Adjust comment about (now-removed) lazy iterators * Pass correct metric name into getMetricNameMatrix() * Pass in go-kit logger to alerting rule creation * Fix ErrNoMetricNameNotSupported message Only the v8 schema supports omitting the metric name in a query. * Expand and improve chunk store tests and fix uncovered bugs - Fix over-selection bug when the metric name was non-equals-matched and one selects a series that has a subset of labels of another. - Regex matches against the metric name were broken because the metric name was equals-compared to the matcher regex instead of applying it as a regex. - Unneeded sorting of label matchers has been removed. * Add back -log.level flag to ingester as well * Ensure sorting of series labels during creation * Add test for querier label sorting * Remove SchemaConfig dependency from ingester The ingester only needs the MaxChunkAge from the SchemaConfig, and the ingester.Config is actually a better place to store and configure that information authoritatively. * Resolve build problems after master rebase * Reparse AM fallback config every time so we can mutate it The deep copy that dumped it as JSON and loaded it again stumbled over the fact that the JSON marshaling renders Secret fields as <secret> and thus loses the original secret field contents when reloading it from JSON. * Fix flaky chunk test
1 parent c3c9968 commit d714b62

File tree

2,444 files changed

+450006
-434143
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,444 files changed

+450006
-434143
lines changed

‎Gopkg.lock

Lines changed: 118 additions & 36 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Gopkg.toml

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,11 @@
44

55
[[constraint]]
66
name = "github.com/prometheus/alertmanager"
7-
revision = "0ecc59076ca6b4cbb63252fa7720a3d89d1c81d3"
7+
revision = "4a8e71069177280d0317ccd80a8201a06f2d8139"
8+
9+
[[constraint]]
10+
name = "github.com/prometheus/prometheus"
11+
branch = "master"
812

913
[[constraint]]
1014
name = "github.com/weaveworks/common"
@@ -14,13 +18,21 @@
1418
name = "github.com/weaveworks/promrus"
1519
version = "v1.1.0-legacy"
1620

21+
[[override]]
22+
name = "k8s.io/client-go"
23+
revision = "3627aeb7d4f6ade38f995d2c923e459146493c7e"
24+
25+
[[override]]
26+
name = "k8s.io/apimachinery"
27+
revision = "75b8dd260ef0469d96d578705a87cffd0e09dab8"
28+
1729
[[override]]
1830
name = "github.com/Azure/azure-sdk-for-go"
1931
revision = "bd73d950fa4440dae889bd9917bff7cef539f86e"
2032

2133
[[override]]
2234
name = "github.com/prometheus/common"
23-
revision = "49fee292b27bfff7f354ee0f64e1bc4850462edf"
35+
revision = "1bab55dd05dbff384524a6a1c99006d9eb5f139b"
2436

2537
[[override]]
2638
name = "github.com/weaveworks/mesh"

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ shell: build-image/$(UPTODATE)
106106
bash
107107

108108
configs-integration-test:
109-
/bin/bash -c "go test -tags netgo,integration -timeout 30s ./pkg/configs/..."
109+
/bin/bash -c "go test -tags 'netgo integration' -timeout 30s ./pkg/configs/..."
110110

111111
endif
112112

‎build-image/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.8.3-stretch
1+
FROM golang:1.9.1-stretch
22
RUN apt-get update && apt-get install -y python-requests python-yaml file jq unzip protobuf-compiler libprotobuf-dev && \
33
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
44
RUN go clean -i net && \

‎cmd/alertmanager/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@ go_library(
88
deps = [
99
"//pkg/alertmanager:go_default_library",
1010
"//pkg/util:go_default_library",
11+
"//vendor/github.com/prometheus/common/log:go_default_library",
1112
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
1213
"//vendor/github.com/weaveworks/common/server:go_default_library",
14+
"//vendor/github.com/weaveworks/promrus:go_default_library",
1315
"//vendor/google.golang.org/grpc:go_default_library",
1416
],
1517
)

‎cmd/alertmanager/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func main() {
3636
}
3737
alertmanagerConfig alertmanager.MultitenantAlertmanagerConfig
3838
)
39-
util.RegisterFlags(&serverConfig, &alertmanagerConfig)
39+
util.RegisterFlags(&serverConfig, &alertmanagerConfig, util.LogLevel{})
4040
flag.Parse()
4141

4242
hook := promrus.MustNewPrometheusHook()

‎cmd/configs/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ go_library(
1212
"//vendor/github.com/prometheus/common/log:go_default_library",
1313
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
1414
"//vendor/github.com/weaveworks/common/server:go_default_library",
15+
"//vendor/github.com/weaveworks/promrus:go_default_library",
1516
"//vendor/google.golang.org/grpc:go_default_library",
1617
],
1718
)

‎cmd/distributor/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ go_library(
1313
"//vendor/github.com/prometheus/common/log:go_default_library",
1414
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
1515
"//vendor/github.com/weaveworks/common/server:go_default_library",
16+
"//vendor/github.com/weaveworks/promrus:go_default_library",
1617
"//vendor/google.golang.org/grpc:go_default_library",
1718
],
1819
)

‎cmd/distributor/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func main() {
4444
ringConfig ring.Config
4545
distributorConfig distributor.Config
4646
)
47-
util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig)
47+
util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, util.LogLevel{})
4848
flag.Parse()
4949

5050
log.AddHook(promrus.MustNewPrometheusHook())

‎cmd/ingester/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ go_library(
1515
"//vendor/github.com/prometheus/common/log:go_default_library",
1616
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
1717
"//vendor/github.com/weaveworks/common/server:go_default_library",
18+
"//vendor/github.com/weaveworks/promrus:go_default_library",
1819
"//vendor/google.golang.org/grpc:go_default_library",
1920
],
2021
)

‎cmd/ingester/main.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ func main() {
3434
// Ingester needs to know our gRPC listen port.
3535
ingesterConfig.ListenPort = &serverConfig.GRPCListenPort
3636
util.RegisterFlags(&serverConfig, &chunkStoreConfig, &storageConfig,
37-
&schemaConfig, &ingesterConfig)
37+
&schemaConfig, &ingesterConfig, util.LogLevel{})
3838
flag.Parse()
39+
schemaConfig.MaxChunkAge = ingesterConfig.MaxChunkAge
3940

4041
log.AddHook(promrus.MustNewPrometheusHook())
4142

@@ -56,7 +57,7 @@ func main() {
5657
}
5758
defer chunkStore.Stop()
5859

59-
ingester, err := ingester.New(ingesterConfig, schemaConfig, chunkStore)
60+
ingester, err := ingester.New(ingesterConfig, chunkStore)
6061
if err != nil {
6162
log.Fatal(err)
6263
}

‎cmd/lite/BUILD.bazel

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ go_library(
1818
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
1919
"//vendor/github.com/prometheus/common/log:go_default_library",
2020
"//vendor/github.com/prometheus/common/route:go_default_library",
21+
"//vendor/github.com/prometheus/prometheus/config:go_default_library",
2122
"//vendor/github.com/prometheus/prometheus/promql:go_default_library",
2223
"//vendor/github.com/prometheus/prometheus/web/api/v1:go_default_library",
2324
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
2425
"//vendor/github.com/weaveworks/common/server:go_default_library",
2526
"//vendor/github.com/weaveworks/common/user:go_default_library",
26-
"//vendor/golang.org/x/net/context:go_default_library",
27+
"//vendor/github.com/weaveworks/promrus:go_default_library",
2728
"//vendor/google.golang.org/grpc:go_default_library",
2829
],
2930
)

‎cmd/lite/main.go

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ import (
77
"github.com/prometheus/client_golang/prometheus"
88
"github.com/prometheus/common/log"
99
"github.com/prometheus/common/route"
10+
"github.com/prometheus/prometheus/config"
1011
"github.com/prometheus/prometheus/promql"
1112
"github.com/prometheus/prometheus/web/api/v1"
12-
"golang.org/x/net/context"
1313
"google.golang.org/grpc"
1414

1515
"github.com/weaveworks/common/middleware"
@@ -48,9 +48,10 @@ func main() {
4848
// Ingester needs to know our gRPC listen port.
4949
ingesterConfig.ListenPort = &serverConfig.GRPCListenPort
5050
util.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig,
51-
&ingesterConfig, &rulerConfig, &storageConfig, &schemaConfig)
51+
&ingesterConfig, &rulerConfig, &storageConfig, &schemaConfig, util.LogLevel{})
5252
flag.BoolVar(&unauthenticated, "unauthenticated", false, "Set to true to disable multitenancy.")
5353
flag.Parse()
54+
schemaConfig.MaxChunkAge = ingesterConfig.MaxChunkAge
5455

5556
log.AddHook(promrus.MustNewPrometheusHook())
5657

@@ -85,7 +86,7 @@ func main() {
8586
defer dist.Stop()
8687
prometheus.MustRegister(dist)
8788

88-
ingester, err := ingester.New(ingesterConfig, schemaConfig, chunkStore)
89+
ingester, err := ingester.New(ingesterConfig, chunkStore)
8990
if err != nil {
9091
log.Fatal(err)
9192
}
@@ -118,13 +119,19 @@ func main() {
118119
defer rulerServer.Stop()
119120
}
120121

121-
queryable := querier.NewQueryable(dist, chunkStore)
122-
engine := promql.NewEngine(queryable, nil)
123-
api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable},
124-
querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{})
125-
promRouter := route.New(func(r *http.Request) (context.Context, error) {
126-
return r.Context(), nil
127-
}).WithPrefix("/api/prom/api/v1")
122+
sampleQueryable := querier.NewQueryable(dist, chunkStore, false)
123+
metadataQueryable := querier.NewQueryable(dist, chunkStore, true)
124+
125+
engine := promql.NewEngine(sampleQueryable, nil)
126+
api := v1.NewAPI(
127+
engine,
128+
metadataQueryable,
129+
querier.DummyTargetRetriever{},
130+
querier.DummyAlertmanagerRetriever{},
131+
func() config.Config { return config.Config{} },
132+
func(f http.HandlerFunc) http.HandlerFunc { return f },
133+
)
134+
promRouter := route.New().WithPrefix("/api/prom/api/v1")
128135
api.Register(promRouter)
129136

130137
activeMiddleware := middleware.AuthenticateUser
@@ -139,7 +146,7 @@ func main() {
139146

140147
subrouter := server.HTTP.PathPrefix("/api/prom").Subrouter()
141148
subrouter.PathPrefix("/api/v1").Handler(activeMiddleware.Wrap(promRouter))
142-
subrouter.Path("/read").Handler(activeMiddleware.Wrap(http.HandlerFunc(queryable.Q.RemoteReadHandler)))
149+
subrouter.Path("/read").Handler(activeMiddleware.Wrap(http.HandlerFunc(sampleQueryable.RemoteReadHandler)))
143150
subrouter.Path("/validate_expr").Handler(activeMiddleware.Wrap(http.HandlerFunc(dist.ValidateExprHandler)))
144151
subrouter.Path("/user_stats").Handler(activeMiddleware.Wrap(http.HandlerFunc(dist.UserStatsHandler)))
145152

‎cmd/querier/BUILD.bazel

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@ go_library(
1515
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
1616
"//vendor/github.com/prometheus/common/log:go_default_library",
1717
"//vendor/github.com/prometheus/common/route:go_default_library",
18+
"//vendor/github.com/prometheus/prometheus/config:go_default_library",
1819
"//vendor/github.com/prometheus/prometheus/promql:go_default_library",
1920
"//vendor/github.com/prometheus/prometheus/web/api/v1:go_default_library",
2021
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
2122
"//vendor/github.com/weaveworks/common/server:go_default_library",
22-
"//vendor/golang.org/x/net/context:go_default_library",
23+
"//vendor/github.com/weaveworks/promrus:go_default_library",
2324
"//vendor/google.golang.org/grpc:go_default_library",
2425
],
2526
)

‎cmd/querier/main.go

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@ import (
44
"flag"
55
"net/http"
66

7-
"golang.org/x/net/context"
87
"google.golang.org/grpc"
98

109
"github.com/prometheus/client_golang/prometheus"
1110
"github.com/prometheus/common/log"
1211
"github.com/prometheus/common/route"
12+
"github.com/prometheus/prometheus/config"
1313
"github.com/prometheus/prometheus/promql"
1414
"github.com/prometheus/prometheus/web/api/v1"
1515

@@ -39,7 +39,7 @@ func main() {
3939
storageConfig storage.Config
4040
)
4141
util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig,
42-
&chunkStoreConfig, &schemaConfig, &storageConfig)
42+
&chunkStoreConfig, &schemaConfig, &storageConfig, util.LogLevel{})
4343
flag.Parse()
4444

4545
log.AddHook(promrus.MustNewPrometheusHook())
@@ -75,18 +75,24 @@ func main() {
7575
}
7676
defer chunkStore.Stop()
7777

78-
queryable := querier.NewQueryable(dist, chunkStore)
79-
engine := promql.NewEngine(queryable, nil)
80-
api := v1.NewAPI(engine, querier.DummyStorage{Queryable: queryable},
81-
querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{})
82-
promRouter := route.New(func(r *http.Request) (context.Context, error) {
83-
return r.Context(), nil
84-
}).WithPrefix("/api/prom/api/v1")
78+
sampleQueryable := querier.NewQueryable(dist, chunkStore, false)
79+
metadataQueryable := querier.NewQueryable(dist, chunkStore, true)
80+
81+
engine := promql.NewEngine(sampleQueryable, nil)
82+
api := v1.NewAPI(
83+
engine,
84+
metadataQueryable,
85+
querier.DummyTargetRetriever{},
86+
querier.DummyAlertmanagerRetriever{},
87+
func() config.Config { return config.Config{} },
88+
func(f http.HandlerFunc) http.HandlerFunc { return f },
89+
)
90+
promRouter := route.New().WithPrefix("/api/prom/api/v1")
8591
api.Register(promRouter)
8692

8793
subrouter := server.HTTP.PathPrefix("/api/prom").Subrouter()
8894
subrouter.PathPrefix("/api/v1").Handler(middleware.AuthenticateUser.Wrap(promRouter))
89-
subrouter.Path("/read").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(queryable.Q.RemoteReadHandler)))
95+
subrouter.Path("/read").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(sampleQueryable.RemoteReadHandler)))
9096
subrouter.Path("/validate_expr").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(dist.ValidateExprHandler)))
9197
subrouter.Path("/user_stats").Handler(middleware.AuthenticateUser.Wrap(http.HandlerFunc(dist.UserStatsHandler)))
9298

‎cmd/ruler/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ go_library(
1616
"//vendor/github.com/prometheus/common/log:go_default_library",
1717
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
1818
"//vendor/github.com/weaveworks/common/server:go_default_library",
19+
"//vendor/github.com/weaveworks/promrus:go_default_library",
1920
"//vendor/google.golang.org/grpc:go_default_library",
2021
],
2122
)

‎cmd/table-manager/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ go_library(
1212
"//vendor/github.com/prometheus/common/log:go_default_library",
1313
"//vendor/github.com/weaveworks/common/middleware:go_default_library",
1414
"//vendor/github.com/weaveworks/common/server:go_default_library",
15+
"//vendor/github.com/weaveworks/promrus:go_default_library",
1516
"//vendor/google.golang.org/grpc:go_default_library",
1617
],
1718
)

‎pkg/alertmanager/BUILD.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@ go_library(
2828
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
2929
"//vendor/github.com/prometheus/common/log:go_default_library",
3030
"//vendor/github.com/prometheus/common/route:go_default_library",
31+
"//vendor/github.com/prometheus/prometheus/pkg/labels:go_default_library",
3132
"//vendor/github.com/weaveworks/common/instrument:go_default_library",
3233
"//vendor/github.com/weaveworks/common/user:go_default_library",
3334
"//vendor/github.com/weaveworks/mesh:go_default_library",
34-
"//vendor/golang.org/x/net/context:go_default_library",
3535
],
3636
)

‎pkg/alertmanager/alertmanager.go

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/prometheus/client_golang/prometheus"
2424
"github.com/prometheus/common/log"
2525
"github.com/prometheus/common/route"
26+
"github.com/prometheus/prometheus/pkg/labels"
2627
"github.com/weaveworks/mesh"
2728
)
2829

@@ -108,16 +109,23 @@ func New(cfg *Config) (*Alertmanager, error) {
108109
am.wg.Done()
109110
}()
110111

111-
am.alerts, err = mem.NewAlerts(cfg.DataDir)
112+
marker := types.NewMarker()
113+
am.alerts, err = mem.NewAlerts(marker, 30*time.Minute, cfg.DataDir)
112114
if err != nil {
113115
return nil, fmt.Errorf("failed to create alerts: %v", err)
114116
}
115117

116-
am.api = api.New(am.alerts, am.silences, func() dispatch.AlertOverview {
117-
return am.dispatcher.Groups()
118-
})
118+
am.api = api.New(
119+
am.alerts,
120+
am.silences,
121+
func(matchers []*labels.Matcher) dispatch.AlertOverview {
122+
return am.dispatcher.Groups(matchers)
123+
},
124+
marker.Status,
125+
nil, // Passing a nil mesh router since we don't show mesh router information in Cortex anyway.
126+
)
119127

120-
am.router = route.New(nil)
128+
am.router = route.New()
121129

122130
webReload := make(chan struct{})
123131
ui.Register(am.router.WithPrefix(am.cfg.ExternalURL.Path), webReload)
@@ -157,7 +165,7 @@ func (am *Alertmanager) ApplyConfig(conf *config.Config) error {
157165
}
158166
tmpl.ExternalURL = am.cfg.ExternalURL
159167

160-
err = am.api.Update(conf.String(), time.Duration(conf.Global.ResolveTimeout))
168+
err = am.api.Update(conf, time.Duration(conf.Global.ResolveTimeout))
161169
if err != nil {
162170
return err
163171
}

0 commit comments

Comments
 (0)