From e9760c0f9776df9d9316cb864f1ec6c7862335a6 Mon Sep 17 00:00:00 2001 From: Aaron Kirkbride Date: Wed, 21 Mar 2018 15:58:48 +0000 Subject: [PATCH 1/9] Update prometheus to latest --- Gopkg.lock | 10 +- .../github.com/json-iterator/go/LICENSE~HEAD | 21 +++ .../go/LICENSE~Update prometheus to latest | 21 +++ .../prometheus/prometheus/config/config.go | 3 + .../prometheus/discovery/azure/azure.go | 5 + .../prometheus/discovery/dns/dns.go | 13 +- .../prometheus/discovery/ec2/ec2.go | 18 +-- .../prometheus/discovery/file/file.go | 2 +- .../discovery/kubernetes/endpoints.go | 15 +- .../prometheus/discovery/kubernetes/pod.go | 10 +- .../prometheus/discovery/manager.go | 10 +- .../discovery/openstack/openstack.go | 3 + .../discovery/targetgroup/targetgroup.go | 6 +- .../prometheus/notifier/notifier.go | 52 +++++-- .../prometheus/pkg/labels/labels.go | 10 ++ .../prometheus/prometheus/pkg/pool/pool.go | 44 ++++-- .../prometheus/prometheus/pkg/textparse/lex.l | 9 +- .../prometheus/pkg/textparse/lex.l.go | 85 ++++++----- .../prometheus/prometheus/prompb/remote.pb.go | 18 --- .../prometheus/prometheus/prompb/rpc.pb.go | 135 +++++++++-------- .../prometheus/prometheus/prompb/rpc.pb.gw.go | 14 +- .../prometheus/prometheus/prompb/rpc.proto | 1 + .../prometheus/prometheus/prompb/types.pb.go | 32 +--- .../prometheus/prometheus/promql/ast.go | 60 ++++---- .../prometheus/prometheus/promql/engine.go | 133 +++++++++-------- .../prometheus/prometheus/promql/lex.go | 32 ++-- .../prometheus/prometheus/promql/parse.go | 25 ++-- .../prometheus/prometheus/promql/test.go | 33 ++-- .../prometheus/prometheus/rules/alerting.go | 2 +- .../prometheus/prometheus/rules/manager.go | 8 +- .../prometheus/prometheus/scrape/manager.go | 13 ++ .../prometheus/prometheus/scrape/scrape.go | 19 ++- .../prometheus/prometheus/scrape/target.go | 8 +- .../prometheus/prometheus/storage/buffer.go | 1 + .../prometheus/prometheus/storage/fanout.go | 4 +- .../prometheus/storage/interface.go | 8 +- .../prometheus/prometheus/storage/noop.go | 2 +- .../storage/remote/queue_manager.go | 23 ++- .../prometheus/storage/remote/read.go | 10 +- .../prometheus/storage/remote/storage.go | 3 +- .../prometheus/storage/tsdb/tsdb.go | 4 +- .../prometheus/template/template.go | 3 +- .../prometheus/util/testutil/error.go | 6 +- .../prometheus/prometheus/web/api/v1/api.go | 141 +++++++++++++----- .../{ => fsnotify}/fsnotify.v1/.editorconfig | 0 .../{ => fsnotify}/fsnotify.v1/.gitignore | 0 .../{ => fsnotify}/fsnotify.v1/.travis.yml | 4 +- .../{ => fsnotify}/fsnotify.v1/AUTHORS | 6 + .../{ => fsnotify}/fsnotify.v1/CHANGELOG.md | 16 +- .../fsnotify.v1/CONTRIBUTING.md | 6 +- .../{ => fsnotify}/fsnotify.v1/LICENSE | 0 .../{ => fsnotify}/fsnotify.v1/README.md | 37 ++++- .../{ => fsnotify}/fsnotify.v1/fen.go | 0 .../{ => fsnotify}/fsnotify.v1/fsnotify.go | 4 + .../{ => fsnotify}/fsnotify.v1/inotify.go | 66 ++++---- .../fsnotify.v1/inotify_poller.go | 0 .../{ => fsnotify}/fsnotify.v1/kqueue.go | 62 +++++--- .../fsnotify.v1/open_mode_bsd.go | 0 .../fsnotify.v1/open_mode_darwin.go | 0 .../{ => fsnotify}/fsnotify.v1/windows.go | 0 60 files changed, 780 insertions(+), 496 deletions(-) create mode 100644 vendor/github.com/json-iterator/go/LICENSE~HEAD create mode 100644 vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/.editorconfig (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/.gitignore (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/.travis.yml (90%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/AUTHORS (88%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/CHANGELOG.md (93%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/CONTRIBUTING.md (93%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/LICENSE (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/README.md (59%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/fen.go (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/fsnotify.go (91%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/inotify.go (87%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/inotify_poller.go (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/kqueue.go (94%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/open_mode_bsd.go (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/open_mode_darwin.go (100%) rename vendor/gopkg.in/{ => fsnotify}/fsnotify.v1/windows.go (100%) diff --git a/Gopkg.lock b/Gopkg.lock index ea8b7007fd..13dc0eea60 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -737,7 +737,7 @@ "util/yaml", "web/api/v1" ] - revision = "4801573b64b34ea929620e948fc36da006f2f966" + revision = "c47fbcb626acb037d1e0ad11ca923cf3fd3a9bdd" [[projects]] branch = "master" @@ -1043,10 +1043,10 @@ version = "v1.9.1" [[projects]] - name = "gopkg.in/fsnotify.v1" + name = "gopkg.in/fsnotify/fsnotify.v1" packages = ["."] - revision = "629574ca2a5df945712d3079857300b5e4da0236" - version = "v1.4.2" + revision = "c2828203cd70a50dcccfb2761f8b1f8ceef9a8e9" + version = "v1.4.7" [[projects]] name = "gopkg.in/inf.v0" @@ -1190,6 +1190,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "9a993b069c652c8a2645ea8807afc6c3430e5034c4f66af7ca55e5ad236a5872" + inputs-digest = "5ce05e0826d6ffafc0d52bdadf741b4db5dc53d5db655ce87785717e67813146" solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/json-iterator/go/LICENSE~HEAD b/vendor/github.com/json-iterator/go/LICENSE~HEAD new file mode 100644 index 0000000000..2cf4f5ab28 --- /dev/null +++ b/vendor/github.com/json-iterator/go/LICENSE~HEAD @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 json-iterator + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest b/vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest new file mode 100644 index 0000000000..2cf4f5ab28 --- /dev/null +++ b/vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 json-iterator + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/prometheus/prometheus/config/config.go b/vendor/github.com/prometheus/prometheus/config/config.go index 9c80d4f63a..824044f1d4 100644 --- a/vendor/github.com/prometheus/prometheus/config/config.go +++ b/vendor/github.com/prometheus/prometheus/config/config.go @@ -569,6 +569,9 @@ func (c *RelabelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if c.Action == RelabelReplace && !relabelTarget.MatchString(c.TargetLabel) { return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action) } + if c.Action == RelabelLabelMap && !relabelTarget.MatchString(c.Replacement) { + return fmt.Errorf("%q is invalid 'replacement' for %s action", c.Replacement, c.Action) + } if c.Action == RelabelHashMod && !model.LabelName(c.TargetLabel).IsValid() { return fmt.Errorf("%q is invalid 'target_label' for %s action", c.TargetLabel, c.Action) } diff --git a/vendor/github.com/prometheus/prometheus/discovery/azure/azure.go b/vendor/github.com/prometheus/prometheus/discovery/azure/azure.go index 78fc6714db..c7119f948e 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/azure/azure.go +++ b/vendor/github.com/prometheus/prometheus/discovery/azure/azure.go @@ -40,6 +40,7 @@ const ( azureLabelMachineID = azureLabel + "machine_id" azureLabelMachineResourceGroup = azureLabel + "machine_resource_group" azureLabelMachineName = azureLabel + "machine_name" + azureLabelMachineOSType = azureLabel + "machine_os_type" azureLabelMachineLocation = azureLabel + "machine_location" azureLabelMachinePrivateIP = azureLabel + "machine_private_ip" azureLabelMachineTag = azureLabel + "machine_tag_" @@ -85,6 +86,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if err != nil { return err } + if c.SubscriptionID == "" { + return fmt.Errorf("Azure SD configuration requires a subscription_id") + } return yaml_util.CheckOverflow(c.XXX, "azure_sd_config") } @@ -247,6 +251,7 @@ func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { labels := model.LabelSet{ azureLabelMachineID: model.LabelValue(*vm.ID), azureLabelMachineName: model.LabelValue(*vm.Name), + azureLabelMachineOSType: model.LabelValue(vm.Properties.StorageProfile.OsDisk.OsType), azureLabelMachineLocation: model.LabelValue(*vm.Location), azureLabelMachineResourceGroup: model.LabelValue(r.ResourceGroup), } diff --git a/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go b/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go index 709d44cba0..a09be0185a 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go +++ b/vendor/github.com/prometheus/prometheus/discovery/dns/dns.go @@ -295,7 +295,7 @@ func lookupFromAnyServer(name string, qtype uint16, conf *dns.ClientConfig, logg for _, server := range conf.Servers { servAddr := net.JoinHostPort(server, conf.Port) - msg, err := askServerForName(name, qtype, client, servAddr, false) + msg, err := askServerForName(name, qtype, client, servAddr, true) if err != nil { level.Warn(logger).Log("msg", "DNS resolution failed", "server", server, "name", name, "err", err) continue @@ -311,8 +311,8 @@ func lookupFromAnyServer(name string, qtype uint16, conf *dns.ClientConfig, logg } // askServerForName makes a request to a specific DNS server for a specific -// name (and qtype). Retries in the event of response truncation, but -// otherwise just sends back whatever the server gave, whether that be a +// name (and qtype). Retries with TCP in the event of response truncation, +// but otherwise just sends back whatever the server gave, whether that be a // valid-looking response, or an error. func askServerForName(name string, queryType uint16, client *dns.Client, servAddr string, edns bool) (*dns.Msg, error) { msg := &dns.Msg{} @@ -327,10 +327,9 @@ func askServerForName(name string, queryType uint16, client *dns.Client, servAdd if client.Net == "tcp" { return nil, fmt.Errorf("got truncated message on TCP (64kiB limit exceeded?)") } - if edns { // Truncated even though EDNS is used - client.Net = "tcp" - } - return askServerForName(name, queryType, client, servAddr, !edns) + + client.Net = "tcp" + return askServerForName(name, queryType, client, servAddr, false) } if err != nil { return nil, err diff --git a/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go b/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go index 51500e633b..aba03066c2 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go +++ b/vendor/github.com/prometheus/prometheus/discovery/ec2/ec2.go @@ -22,7 +22,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" @@ -186,16 +185,6 @@ func (d *Discovery) Run(ctx context.Context, ch chan<- []*targetgroup.Group) { } } -func (d *Discovery) ec2MetadataAvailable(sess *session.Session) (isAvailable bool) { - svc := ec2metadata.New(sess, &aws.Config{ - MaxRetries: aws.Int(0), - }) - - isAvailable = svc.Available() - - return isAvailable -} - func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { t0 := time.Now() defer func() { @@ -218,12 +207,7 @@ func (d *Discovery) refresh() (tg *targetgroup.Group, err error) { creds := stscreds.NewCredentials(sess, d.roleARN) ec2s = ec2.New(sess, &aws.Config{Credentials: creds}) } else { - if d.aws.Credentials == nil && d.ec2MetadataAvailable(sess) { - creds := ec2rolecreds.NewCredentials(sess) - ec2s = ec2.New(sess, &aws.Config{Credentials: creds}) - } else { - ec2s = ec2.New(sess) - } + ec2s = ec2.New(sess) } tg = &targetgroup.Group{ Source: *d.aws.Region, diff --git a/vendor/github.com/prometheus/prometheus/discovery/file/file.go b/vendor/github.com/prometheus/prometheus/discovery/file/file.go index cf1e1deb00..c2358dfd94 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/file/file.go +++ b/vendor/github.com/prometheus/prometheus/discovery/file/file.go @@ -32,7 +32,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/discovery/targetgroup" yaml_util "github.com/prometheus/prometheus/util/yaml" - "gopkg.in/fsnotify.v1" + "gopkg.in/fsnotify/fsnotify.v1" "gopkg.in/yaml.v2" ) diff --git a/vendor/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go b/vendor/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go index b700399ffc..69ea4ce599 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go +++ b/vendor/github.com/prometheus/prometheus/discovery/kubernetes/endpoints.go @@ -178,10 +178,12 @@ func endpointsSource(ep *apiv1.Endpoints) string { } const ( - endpointsNameLabel = metaLabelPrefix + "endpoints_name" - endpointReadyLabel = metaLabelPrefix + "endpoint_ready" - endpointPortNameLabel = metaLabelPrefix + "endpoint_port_name" - endpointPortProtocolLabel = metaLabelPrefix + "endpoint_port_protocol" + endpointsNameLabel = metaLabelPrefix + "endpoints_name" + endpointReadyLabel = metaLabelPrefix + "endpoint_ready" + endpointPortNameLabel = metaLabelPrefix + "endpoint_port_name" + endpointPortProtocolLabel = metaLabelPrefix + "endpoint_port_protocol" + endpointAddressTargetKindLabel = metaLabelPrefix + "endpoint_address_target_kind" + endpointAddressTargetNameLabel = metaLabelPrefix + "endpoint_address_target_name" ) func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { @@ -210,6 +212,11 @@ func (e *Endpoints) buildEndpoints(eps *apiv1.Endpoints) *targetgroup.Group { endpointReadyLabel: lv(ready), } + if addr.TargetRef != nil { + target[model.LabelName(endpointAddressTargetKindLabel)] = lv(addr.TargetRef.Kind) + target[model.LabelName(endpointAddressTargetNameLabel)] = lv(addr.TargetRef.Name) + } + pod := e.resolvePodRef(addr.TargetRef) if pod == nil { // This target is not a Pod, so don't continue with Pod specific logic. diff --git a/vendor/github.com/prometheus/prometheus/discovery/kubernetes/pod.go b/vendor/github.com/prometheus/prometheus/discovery/kubernetes/pod.go index d868b1156a..2c307a667d 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/kubernetes/pod.go +++ b/vendor/github.com/prometheus/prometheus/discovery/kubernetes/pod.go @@ -170,14 +170,14 @@ func podLabels(pod *apiv1.Pod) model.LabelSet { } func (p *Pod) buildPod(pod *apiv1.Pod) *targetgroup.Group { - // During startup the pod may not have an IP yet. This does not even allow - // for an up metric, so we skip the target. - if len(pod.Status.PodIP) == 0 { - return nil - } tg := &targetgroup.Group{ Source: podSource(pod), } + // PodIP can be empty when a pod is starting or has been evicted. + if len(pod.Status.PodIP) == 0 { + return tg + } + tg.Labels = podLabels(pod) tg.Labels[namespaceLabel] = lv(pod.Namespace) diff --git a/vendor/github.com/prometheus/prometheus/discovery/manager.go b/vendor/github.com/prometheus/prometheus/discovery/manager.go index 0b6716048f..5b5753f6f9 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/manager.go +++ b/vendor/github.com/prometheus/prometheus/discovery/manager.go @@ -90,13 +90,11 @@ type Manager struct { // Run starts the background processing func (m *Manager) Run() error { - for { - select { - case <-m.ctx.Done(): - m.cancelDiscoverers() - return m.ctx.Err() - } + for range m.ctx.Done() { + m.cancelDiscoverers() + return m.ctx.Err() } + return nil } // SyncCh returns a read only channel used by all Discoverers to send target updates. diff --git a/vendor/github.com/prometheus/prometheus/discovery/openstack/openstack.go b/vendor/github.com/prometheus/prometheus/discovery/openstack/openstack.go index 85a6aff100..ac7371db99 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/openstack/openstack.go +++ b/vendor/github.com/prometheus/prometheus/discovery/openstack/openstack.go @@ -103,6 +103,9 @@ func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if c.Role == "" { return fmt.Errorf("role missing (one of: instance, hypervisor)") } + if c.Region == "" { + return fmt.Errorf("Openstack SD configuration requires a region") + } return yaml_util.CheckOverflow(c.XXX, "openstack_sd_config") } diff --git a/vendor/github.com/prometheus/prometheus/discovery/targetgroup/targetgroup.go b/vendor/github.com/prometheus/prometheus/discovery/targetgroup/targetgroup.go index 4bafcef9e2..0197622c2f 100644 --- a/vendor/github.com/prometheus/prometheus/discovery/targetgroup/targetgroup.go +++ b/vendor/github.com/prometheus/prometheus/discovery/targetgroup/targetgroup.go @@ -14,6 +14,7 @@ package targetgroup import ( + "bytes" "encoding/json" "github.com/prometheus/common/model" @@ -77,7 +78,10 @@ func (tg *Group) UnmarshalJSON(b []byte) error { Targets []string `json:"targets"` Labels model.LabelSet `json:"labels"` }{} - if err := json.Unmarshal(b, &g); err != nil { + + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + if err := dec.Decode(&g); err != nil { return err } tg.Targets = make([]model.LabelSet, 0, len(g.Targets)) diff --git a/vendor/github.com/prometheus/prometheus/notifier/notifier.go b/vendor/github.com/prometheus/prometheus/notifier/notifier.go index 94e4c5e7ea..92ecde1b4f 100644 --- a/vendor/github.com/prometheus/prometheus/notifier/notifier.go +++ b/vendor/github.com/prometheus/prometheus/notifier/notifier.go @@ -145,7 +145,7 @@ func newAlertMetrics(r prometheus.Registerer, queueCap int, queueLen, alertmanag Namespace: namespace, Subsystem: subsystem, Name: "latency_seconds", - Help: "Latency quantiles for sending alert notifications (not including dropped notifications).", + Help: "Latency quantiles for sending alert notifications.", }, []string{alertmanagerLabel}, ), @@ -161,7 +161,7 @@ func newAlertMetrics(r prometheus.Registerer, queueCap int, queueLen, alertmanag Namespace: namespace, Subsystem: subsystem, Name: "sent_total", - Help: "Total number of alerts successfully sent.", + Help: "Total number of alerts sent.", }, []string{alertmanagerLabel}, ), @@ -419,6 +419,25 @@ func (n *Manager) Alertmanagers() []*url.URL { return res } +// DroppedAlertmanagers returns a slice of Alertmanager URLs. +func (n *Manager) DroppedAlertmanagers() []*url.URL { + n.mtx.RLock() + amSets := n.alertmanagers + n.mtx.RUnlock() + + var res []*url.URL + + for _, ams := range amSets { + ams.mtx.RLock() + for _, dam := range ams.droppedAms { + res = append(res, dam.url()) + } + ams.mtx.RUnlock() + } + + return res +} + // sendAll sends the alerts to all configured Alertmanagers concurrently. // It returns true if the alerts could be sent successfully to at least one Alertmanager. func (n *Manager) sendAll(alerts ...*Alert) bool { @@ -519,9 +538,10 @@ type alertmanagerSet struct { metrics *alertMetrics - mtx sync.RWMutex - ams []alertmanager - logger log.Logger + mtx sync.RWMutex + ams []alertmanager + droppedAms []alertmanager + logger log.Logger } func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger) (*alertmanagerSet, error) { @@ -540,24 +560,28 @@ func newAlertmanagerSet(cfg *config.AlertmanagerConfig, logger log.Logger) (*ale // sync extracts a deduplicated set of Alertmanager endpoints from a list // of target groups definitions. func (s *alertmanagerSet) sync(tgs []*targetgroup.Group) { - all := []alertmanager{} + allAms := []alertmanager{} + allDroppedAms := []alertmanager{} for _, tg := range tgs { - ams, err := alertmanagerFromGroup(tg, s.cfg) + ams, droppedAms, err := alertmanagerFromGroup(tg, s.cfg) if err != nil { level.Error(s.logger).Log("msg", "Creating discovered Alertmanagers failed", "err", err) continue } - all = append(all, ams...) + allAms = append(allAms, ams...) + allDroppedAms = append(allDroppedAms, droppedAms...) } s.mtx.Lock() defer s.mtx.Unlock() // Set new Alertmanagers and deduplicate them along their unique URL. s.ams = []alertmanager{} + s.droppedAms = []alertmanager{} + s.droppedAms = append(s.droppedAms, allDroppedAms...) seen := map[string]struct{}{} - for _, am := range all { + for _, am := range allAms { us := am.url().String() if _, ok := seen[us]; ok { continue @@ -578,8 +602,9 @@ func postPath(pre string) string { // alertmanagersFromGroup extracts a list of alertmanagers from a target group and an associcated // AlertmanagerConfig. -func alertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig) ([]alertmanager, error) { +func alertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig) ([]alertmanager, []alertmanager, error) { var res []alertmanager + var droppedAlertManagers []alertmanager for _, tlset := range tg.Targets { lbls := make([]labels.Label, 0, len(tlset)+2+len(tg.Labels)) @@ -600,6 +625,7 @@ func alertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig lset := relabel.Process(labels.New(lbls...), cfg.RelabelConfigs...) if lset == nil { + droppedAlertManagers = append(droppedAlertManagers, alertmanagerLabels{lbls}) continue } @@ -627,13 +653,13 @@ func alertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig case "https": addr = addr + ":443" default: - return nil, fmt.Errorf("invalid scheme: %q", cfg.Scheme) + return nil, nil, fmt.Errorf("invalid scheme: %q", cfg.Scheme) } lb.Set(model.AddressLabel, addr) } if err := config.CheckTargetAddress(model.LabelValue(addr)); err != nil { - return nil, err + return nil, nil, err } // Meta labels are deleted after relabelling. Other internal labels propagate to @@ -646,5 +672,5 @@ func alertmanagerFromGroup(tg *targetgroup.Group, cfg *config.AlertmanagerConfig res = append(res, alertmanagerLabels{lset}) } - return res, nil + return res, droppedAlertManagers, nil } diff --git a/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go b/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go index 471677dbc2..0fbdce36bd 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go +++ b/vendor/github.com/prometheus/prometheus/pkg/labels/labels.go @@ -112,6 +112,16 @@ func (ls Labels) Get(name string) string { return "" } +// Has returns true if the label with the given name is present. +func (ls Labels) Has(name string) bool { + for _, l := range ls { + if l.Name == name { + return true + } + } + return false +} + // Equal returns whether the two label sets are equal. func Equal(ls, o Labels) bool { if len(ls) != len(o) { diff --git a/vendor/github.com/prometheus/prometheus/pkg/pool/pool.go b/vendor/github.com/prometheus/prometheus/pkg/pool/pool.go index 7cfa78f42c..2ee8971854 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/pool/pool.go +++ b/vendor/github.com/prometheus/prometheus/pkg/pool/pool.go @@ -13,17 +13,23 @@ package pool -import "sync" +import ( + "fmt" + "reflect" + "sync" +) -// BytesPool is a bucketed pool for variably sized byte slices. -type BytesPool struct { +// Pool is a bucketed pool for variably sized byte slices. +type Pool struct { buckets []sync.Pool sizes []int + // make is the function used to create an empty slice when none exist yet. + make func(int) interface{} } -// NewBytesPool returns a new BytesPool with size buckets for minSize to maxSize +// New returns a new Pool with size buckets for minSize to maxSize // increasing by the given factor. -func NewBytesPool(minSize, maxSize int, factor float64) *BytesPool { +func New(minSize, maxSize int, factor float64, makeFunc func(int) interface{}) *Pool { if minSize < 1 { panic("invalid minimum pool size") } @@ -40,36 +46,42 @@ func NewBytesPool(minSize, maxSize int, factor float64) *BytesPool { sizes = append(sizes, s) } - p := &BytesPool{ + p := &Pool{ buckets: make([]sync.Pool, len(sizes)), sizes: sizes, + make: makeFunc, } return p } // Get returns a new byte slices that fits the given size. -func (p *BytesPool) Get(sz int) []byte { +func (p *Pool) Get(sz int) interface{} { for i, bktSize := range p.sizes { if sz > bktSize { continue } - b, ok := p.buckets[i].Get().([]byte) - if !ok { - b = make([]byte, 0, bktSize) + b := p.buckets[i].Get() + if b == nil { + b = p.make(bktSize) } return b } - return make([]byte, 0, sz) + return p.make(sz) } -// Put returns a byte slice to the right bucket in the pool. -func (p *BytesPool) Put(b []byte) { - for i, bktSize := range p.sizes { - if cap(b) > bktSize { +// Put adds a slice to the right bucket in the pool. +func (p *Pool) Put(s interface{}) { + slice := reflect.ValueOf(s) + + if slice.Kind() != reflect.Slice { + panic(fmt.Sprintf("%+v is not a slice", slice)) + } + for i, size := range p.sizes { + if slice.Cap() > size { continue } - p.buckets[i].Put(b[:0]) + p.buckets[i].Put(slice.Slice(0, 0).Interface()) return } } diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l b/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l index 7d15bfd212..cc711e4105 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l @@ -30,6 +30,7 @@ const ( lstateTimestamp lstateLabels lstateLName + lstateLEq lstateLValue lstateLValueIn ) @@ -54,7 +55,7 @@ D [0-9] L [a-zA-Z_] M [a-zA-Z_:] -%x lstateName lstateValue lstateTimestamp lstateLabels lstateLName lstateLValue lstateLValueIn +%x lstateName lstateValue lstateTimestamp lstateLabels lstateLName lstateLEq lstateLValue lstateLValueIn %yyc c @@ -84,8 +85,10 @@ M [a-zA-Z_:] (,?[ \t]*) l.state = lstateLName l.offsets = append(l.offsets, l.i) -{L}({L}|{D})* l.offsets = append(l.offsets, l.i) -[ \t]*= l.state = lstateLValue +{L}({L}|{D})* l.state = lstateLEq + l.offsets = append(l.offsets, l.i) + +[ \t]*= l.state = lstateLValue [ \t]+ \" l.state = lstateLValueIn diff --git a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go b/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go index 8486e762b4..0a45d070f1 100644 --- a/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go +++ b/vendor/github.com/prometheus/prometheus/pkg/textparse/lex.l.go @@ -31,6 +31,7 @@ const ( lstateTimestamp lstateLabels lstateLName + lstateLEq lstateLValue lstateLValueIn ) @@ -67,10 +68,12 @@ yystate0: goto yystart21 case 5: // start condition: lstateLName goto yystart26 - case 6: // start condition: lstateLValue - goto yystart30 - case 7: // start condition: lstateLValueIn - goto yystart33 + case 6: // start condition: lstateLEq + goto yystart28 + case 7: // start condition: lstateLValue + goto yystart31 + case 8: // start condition: lstateLValueIn + goto yystart34 } goto yystate0 // silence unused label error @@ -323,103 +326,112 @@ yystart26: switch { default: goto yyabort - case c == '=': - goto yystate28 - case c == '\t' || c == ' ': - goto yystate27 case c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': - goto yystate29 + goto yystate27 } yystate27: c = l.next() switch { default: - goto yyabort - case c == '=': - goto yystate28 - case c == '\t' || c == ' ': + goto yyrule10 + case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': goto yystate27 } + goto yystate28 // silence unused label error yystate28: c = l.next() - goto yyrule11 +yystart28: + switch { + default: + goto yyabort + case c == '=': + goto yystate30 + case c == '\t' || c == ' ': + goto yystate29 + } yystate29: c = l.next() switch { default: - goto yyrule10 - case c >= '0' && c <= '9' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z': + goto yyabort + case c == '=': + goto yystate30 + case c == '\t' || c == ' ': goto yystate29 } - goto yystate30 // silence unused label error yystate30: c = l.next() -yystart30: + goto yyrule11 + + goto yystate31 // silence unused label error +yystate31: + c = l.next() +yystart31: switch { default: goto yyabort case c == '"': - goto yystate32 + goto yystate33 case c == '\t' || c == ' ': - goto yystate31 + goto yystate32 } -yystate31: +yystate32: c = l.next() switch { default: goto yyrule12 case c == '\t' || c == ' ': - goto yystate31 + goto yystate32 } -yystate32: +yystate33: c = l.next() goto yyrule13 - goto yystate33 // silence unused label error -yystate33: + goto yystate34 // silence unused label error +yystate34: c = l.next() -yystart33: +yystart34: switch { default: goto yyabort case c == '"': - goto yystate35 - case c == '\\': goto yystate36 + case c == '\\': + goto yystate37 case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': - goto yystate34 + goto yystate35 } -yystate34: +yystate35: c = l.next() switch { default: goto yyabort case c == '"': - goto yystate35 - case c == '\\': goto yystate36 + case c == '\\': + goto yystate37 case c >= '\x01' && c <= '!' || c >= '#' && c <= '[' || c >= ']' && c <= 'ÿ': - goto yystate34 + goto yystate35 } -yystate35: +yystate36: c = l.next() goto yyrule14 -yystate36: +yystate37: c = l.next() switch { default: goto yyabort case c >= '\x01' && c <= '\t' || c >= '\v' && c <= 'ÿ': - goto yystate34 + goto yystate35 } yyrule1: // \0 @@ -471,6 +483,7 @@ yyrule9: // (,?[ \t]*) } yyrule10: // {L}({L}|{D})* { + l.state = lstateLEq l.offsets = append(l.offsets, l.i) goto yystate0 } diff --git a/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go b/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go index ad91fa5b9a..e7ad9d76de 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go +++ b/vendor/github.com/prometheus/prometheus/prompb/remote.pb.go @@ -311,24 +311,6 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Remote(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Remote(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintRemote(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) diff --git a/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.go b/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.go index 40c3e94368..e8ad8981c9 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.go +++ b/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.go @@ -12,12 +12,10 @@ import _ "github.com/gogo/protobuf/gogoproto" import time "time" -import ( - context "golang.org/x/net/context" - grpc "google.golang.org/grpc" -) +import context "golang.org/x/net/context" +import grpc "google.golang.org/grpc" -import github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" +import types "github.com/gogo/protobuf/types" import io "io" @@ -28,6 +26,7 @@ var _ = math.Inf var _ = time.Kitchen type TSDBSnapshotRequest struct { + SkipHead bool `protobuf:"varint,1,opt,name=skip_head,json=skipHead,proto3" json:"skip_head,omitempty"` } func (m *TSDBSnapshotRequest) Reset() { *m = TSDBSnapshotRequest{} } @@ -247,6 +246,16 @@ func (m *TSDBSnapshotRequest) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SkipHead { + dAtA[i] = 0x8 + i++ + if m.SkipHead { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -328,8 +337,8 @@ func (m *SeriesDeleteRequest) MarshalTo(dAtA []byte) (int, error) { if m.MinTime != nil { dAtA[i] = 0xa i++ - i = encodeVarintRpc(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.MinTime))) - n1, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MinTime, dAtA[i:]) + i = encodeVarintRpc(dAtA, i, uint64(types.SizeOfStdTime(*m.MinTime))) + n1, err := types.StdTimeMarshalTo(*m.MinTime, dAtA[i:]) if err != nil { return 0, err } @@ -338,8 +347,8 @@ func (m *SeriesDeleteRequest) MarshalTo(dAtA []byte) (int, error) { if m.MaxTime != nil { dAtA[i] = 0x12 i++ - i = encodeVarintRpc(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxTime))) - n2, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(*m.MaxTime, dAtA[i:]) + i = encodeVarintRpc(dAtA, i, uint64(types.SizeOfStdTime(*m.MaxTime))) + n2, err := types.StdTimeMarshalTo(*m.MaxTime, dAtA[i:]) if err != nil { return 0, err } @@ -378,24 +387,6 @@ func (m *SeriesDeleteResponse) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Rpc(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Rpc(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -408,6 +399,9 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int { func (m *TSDBSnapshotRequest) Size() (n int) { var l int _ = l + if m.SkipHead { + n += 2 + } return n } @@ -437,11 +431,11 @@ func (m *SeriesDeleteRequest) Size() (n int) { var l int _ = l if m.MinTime != nil { - l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.MinTime) + l = types.SizeOfStdTime(*m.MinTime) n += 1 + l + sovRpc(uint64(l)) } if m.MaxTime != nil { - l = github_com_gogo_protobuf_types.SizeOfStdTime(*m.MaxTime) + l = types.SizeOfStdTime(*m.MaxTime) n += 1 + l + sovRpc(uint64(l)) } if len(m.Matchers) > 0 { @@ -501,6 +495,26 @@ func (m *TSDBSnapshotRequest) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: TSDBSnapshotRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SkipHead", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.SkipHead = bool(v != 0) default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) @@ -759,7 +773,7 @@ func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error { if m.MinTime == nil { m.MinTime = new(time.Time) } - if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.MinTime, dAtA[iNdEx:postIndex]); err != nil { + if err := types.StdTimeUnmarshal(m.MinTime, dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -792,7 +806,7 @@ func (m *SeriesDeleteRequest) Unmarshal(dAtA []byte) error { if m.MaxTime == nil { m.MaxTime = new(time.Time) } - if err := github_com_gogo_protobuf_types.StdTimeUnmarshal(m.MaxTime, dAtA[iNdEx:postIndex]); err != nil { + if err := types.StdTimeUnmarshal(m.MaxTime, dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -1006,34 +1020,35 @@ var ( func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) } var fileDescriptorRpc = []byte{ - // 451 bytes of a gzipped FileDescriptorProto + // 471 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x3d, 0x8f, 0xd3, 0x40, - 0x10, 0xbd, 0xbd, 0x84, 0x83, 0xdb, 0x5c, 0xe5, 0x0b, 0x10, 0x4c, 0x88, 0x83, 0x0b, 0xee, 0x74, - 0x85, 0x57, 0x0a, 0xdd, 0x51, 0x11, 0xae, 0x84, 0x26, 0x49, 0x45, 0x13, 0xad, 0x93, 0xc1, 0xb1, - 0xe4, 0xfd, 0xc0, 0xbb, 0x41, 0x07, 0x25, 0x1d, 0x15, 0x48, 0xfc, 0xa9, 0x48, 0x34, 0x48, 0xf4, - 0x7c, 0x44, 0xfc, 0x10, 0xb4, 0xbb, 0xf6, 0x5d, 0x6c, 0x19, 0x41, 0x37, 0x1e, 0xbf, 0x37, 0x6f, - 0xde, 0x9b, 0xc5, 0x87, 0xb9, 0x5c, 0x44, 0x32, 0x17, 0x5a, 0x78, 0x58, 0xe6, 0x82, 0x81, 0x5e, - 0xc1, 0x5a, 0xf9, 0x1d, 0xfd, 0x56, 0x82, 0x72, 0x3f, 0xfc, 0x20, 0x11, 0x22, 0xc9, 0x80, 0xd8, - 0xaf, 0x78, 0xfd, 0x8a, 0xe8, 0x94, 0x81, 0xd2, 0x94, 0xc9, 0x02, 0xd0, 0x2f, 0x00, 0x54, 0xa6, - 0x84, 0x72, 0x2e, 0x34, 0xd5, 0xa9, 0xe0, 0x25, 0xbd, 0x9b, 0x88, 0x44, 0xd8, 0x92, 0x98, 0xca, - 0x75, 0xc3, 0xdb, 0xf8, 0x78, 0x36, 0xbd, 0x18, 0x4f, 0x39, 0x95, 0x6a, 0x25, 0xf4, 0x04, 0x5e, - 0xaf, 0x41, 0xe9, 0xf0, 0x0c, 0x77, 0xab, 0x6d, 0x25, 0x05, 0x57, 0xe0, 0x79, 0xb8, 0xcd, 0x29, - 0x83, 0x1e, 0x1a, 0xa2, 0xd3, 0xc3, 0x89, 0xad, 0xc3, 0x3e, 0xf6, 0x0d, 0xf6, 0x59, 0x06, 0x94, - 0xcf, 0x04, 0x8b, 0x95, 0x16, 0x1c, 0x54, 0x39, 0xe9, 0x01, 0xbe, 0xdf, 0xf8, 0xd7, 0x0d, 0x0c, - 0xbf, 0x20, 0x7c, 0x3c, 0x85, 0x3c, 0x05, 0x75, 0x01, 0x19, 0x68, 0x28, 0x68, 0xde, 0x13, 0x7c, - 0x8b, 0xa5, 0x7c, 0x6e, 0x2c, 0x5a, 0xb1, 0xce, 0xc8, 0x8f, 0x9c, 0xbd, 0xa8, 0xf4, 0x1f, 0xcd, - 0x4a, 0xff, 0xe3, 0xf6, 0xa7, 0x1f, 0x01, 0x9a, 0xdc, 0x64, 0x29, 0x37, 0x3d, 0x4b, 0xa6, 0x97, - 0x8e, 0xbc, 0xff, 0xdf, 0x64, 0x7a, 0x69, 0xc9, 0xe7, 0x86, 0xac, 0x17, 0x2b, 0xc8, 0x55, 0xaf, - 0x35, 0x6c, 0x9d, 0x76, 0x46, 0xbd, 0xe8, 0xfa, 0x24, 0xd1, 0x73, 0x1a, 0x43, 0xf6, 0xc2, 0x01, - 0xc6, 0xed, 0xcd, 0xf7, 0x60, 0x6f, 0x72, 0x85, 0x0f, 0xef, 0xe0, 0x6e, 0xd5, 0x8c, 0x73, 0x39, - 0xfa, 0xd0, 0xc2, 0x37, 0x9e, 0x2e, 0x59, 0xca, 0xbd, 0x1c, 0x1f, 0xed, 0x06, 0xeb, 0x05, 0xbb, - 0xb3, 0x1b, 0x2e, 0xe1, 0x0f, 0xff, 0x0e, 0x28, 0x22, 0x0c, 0xde, 0x7f, 0xfb, 0xfd, 0x79, 0xff, - 0x5e, 0x78, 0x97, 0xbc, 0x19, 0x11, 0x6a, 0x54, 0x88, 0x56, 0xcb, 0x98, 0xa8, 0x52, 0xe3, 0x23, - 0x72, 0x47, 0xae, 0xdd, 0xc0, 0x7b, 0x54, 0x1f, 0xdd, 0x7c, 0x42, 0xff, 0xe4, 0x9f, 0xb8, 0x62, - 0x93, 0x13, 0xbb, 0xc9, 0xc3, 0x30, 0xa8, 0x6d, 0xb2, 0x30, 0xf8, 0xb9, 0xbe, 0x56, 0x7e, 0x87, - 0x8f, 0x5c, 0x42, 0x2e, 0xad, 0x6a, 0x0a, 0x0d, 0xcf, 0xa1, 0x9a, 0x42, 0x53, 0xc4, 0x57, 0xda, - 0xfd, 0x9a, 0xf6, 0xd2, 0xc2, 0xe6, 0xca, 0x72, 0xce, 0xd1, 0xd9, 0xb8, 0xb7, 0xf9, 0x35, 0xd8, - 0xdb, 0x6c, 0x07, 0xe8, 0xeb, 0x76, 0x80, 0x7e, 0x6e, 0x07, 0xe8, 0xe5, 0x81, 0x99, 0x2d, 0xe3, - 0xf8, 0xc0, 0x3e, 0x8e, 0xc7, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x6d, 0x79, 0xf1, 0x8d, - 0x03, 0x00, 0x00, + 0x10, 0xbd, 0xbd, 0x84, 0x23, 0xd9, 0x5c, 0xe5, 0x8b, 0x20, 0xf8, 0x42, 0x1c, 0x5c, 0x70, 0xa7, + 0x2b, 0x6c, 0xc9, 0x74, 0x47, 0x45, 0xb8, 0x82, 0x02, 0x1a, 0x27, 0x15, 0x4d, 0xb4, 0x8e, 0x87, + 0xc4, 0x22, 0xfb, 0x81, 0x77, 0x83, 0x0e, 0x4a, 0x3a, 0x2a, 0x90, 0xf8, 0x53, 0x91, 0x68, 0x90, + 0xe8, 0xf9, 0x88, 0xf8, 0x21, 0x68, 0x77, 0xed, 0xbb, 0xc4, 0x32, 0xe2, 0xba, 0xd9, 0xd9, 0xf7, + 0xe6, 0xcd, 0xbc, 0x19, 0xdc, 0xce, 0xc5, 0x2c, 0x10, 0x39, 0x57, 0xdc, 0xc1, 0x22, 0xe7, 0x14, + 0xd4, 0x02, 0x56, 0xd2, 0xed, 0xa8, 0x77, 0x02, 0xa4, 0xfd, 0x70, 0xbd, 0x39, 0xe7, 0xf3, 0x25, + 0x84, 0xe6, 0x95, 0xac, 0x5e, 0x85, 0x2a, 0xa3, 0x20, 0x15, 0xa1, 0xa2, 0x00, 0xf4, 0x0b, 0x00, + 0x11, 0x59, 0x48, 0x18, 0xe3, 0x8a, 0xa8, 0x8c, 0xb3, 0x92, 0xde, 0x9d, 0xf3, 0x39, 0x37, 0x61, + 0xa8, 0x23, 0x9b, 0xf5, 0x23, 0x7c, 0x34, 0x19, 0x5f, 0x8c, 0xc6, 0x8c, 0x08, 0xb9, 0xe0, 0x2a, + 0x86, 0x37, 0x2b, 0x90, 0xca, 0x39, 0xc6, 0x6d, 0xf9, 0x3a, 0x13, 0xd3, 0x05, 0x90, 0xb4, 0x87, + 0x86, 0xe8, 0xb4, 0x15, 0xb7, 0x74, 0xe2, 0x19, 0x90, 0xd4, 0x3f, 0xc3, 0xdd, 0x5d, 0x8e, 0x14, + 0x9c, 0x49, 0x70, 0x1c, 0xdc, 0x64, 0x84, 0x82, 0xc1, 0xb7, 0x63, 0x13, 0xfb, 0x7d, 0xec, 0x6a, + 0xec, 0xd3, 0x25, 0x10, 0x36, 0xe1, 0x34, 0x91, 0x8a, 0x33, 0x90, 0x85, 0x8c, 0x7f, 0x1f, 0x1f, + 0xd7, 0xfe, 0xda, 0x82, 0xfe, 0x57, 0x84, 0x8f, 0xc6, 0x90, 0x67, 0x20, 0x2f, 0x60, 0x09, 0x0a, + 0xca, 0xee, 0x1e, 0xe3, 0x16, 0xcd, 0xd8, 0x54, 0xcf, 0x6f, 0xc4, 0x3a, 0x91, 0x1b, 0xd8, 0xd9, + 0x83, 0xd2, 0x9c, 0x60, 0x52, 0x9a, 0x33, 0x6a, 0x7e, 0xfe, 0xe9, 0xa1, 0xf8, 0x36, 0xcd, 0x98, + 0xce, 0x19, 0x32, 0xb9, 0xb4, 0xe4, 0xfd, 0x1b, 0x93, 0xc9, 0xa5, 0x21, 0x9f, 0x6b, 0xb2, 0x9a, + 0x2d, 0x20, 0x97, 0xbd, 0xc6, 0xb0, 0x71, 0xda, 0x89, 0x7a, 0xc1, 0xf5, 0xbe, 0x82, 0xe7, 0x24, + 0x81, 0xe5, 0x0b, 0x0b, 0x18, 0x35, 0xd7, 0x3f, 0xbc, 0xbd, 0xf8, 0x0a, 0xef, 0xdf, 0xc1, 0xdd, + 0xdd, 0x61, 0xec, 0x94, 0xd1, 0xc7, 0x06, 0xbe, 0xf5, 0x24, 0xa5, 0x19, 0x73, 0x72, 0x7c, 0xb8, + 0x6d, 0xac, 0xe3, 0x6d, 0xd7, 0xae, 0x59, 0x93, 0x3b, 0xfc, 0x37, 0xa0, 0xb0, 0xd0, 0xfb, 0xf0, + 0xfd, 0xcf, 0x97, 0xfd, 0x7b, 0xfe, 0xdd, 0xf0, 0x6d, 0x14, 0x12, 0xad, 0x12, 0x2a, 0x99, 0x26, + 0xa1, 0x2c, 0x35, 0x3e, 0x21, 0x7b, 0x01, 0x95, 0x1d, 0x38, 0x0f, 0xab, 0xa5, 0xeb, 0x57, 0xe8, + 0x9e, 0xfc, 0x17, 0x57, 0x74, 0x72, 0x62, 0x3a, 0x79, 0xe0, 0x7b, 0x95, 0x4e, 0x66, 0x1a, 0x3f, + 0x55, 0xd7, 0xca, 0xef, 0xf1, 0xa1, 0x75, 0xc8, 0xba, 0xb5, 0xeb, 0x42, 0xcd, 0x39, 0xec, 0xba, + 0x50, 0x67, 0xf1, 0x95, 0x76, 0xbf, 0xa2, 0x9d, 0x1a, 0xd8, 0x54, 0x1a, 0xce, 0x39, 0x3a, 0x1b, + 0xf5, 0xd6, 0xbf, 0x07, 0x7b, 0xeb, 0xcd, 0x00, 0x7d, 0xdb, 0x0c, 0xd0, 0xaf, 0xcd, 0x00, 0xbd, + 0x3c, 0xd0, 0xb5, 0x45, 0x92, 0x1c, 0x98, 0xe3, 0x78, 0xf4, 0x37, 0x00, 0x00, 0xff, 0xff, 0x8b, + 0x21, 0x42, 0x5d, 0xaa, 0x03, 0x00, 0x00, } diff --git a/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go b/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go index 177ea4c108..6a43047329 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go +++ b/vendor/github.com/prometheus/prometheus/prompb/rpc.pb.gw.go @@ -28,10 +28,18 @@ var _ status.Status var _ = runtime.String var _ = utilities.NewDoubleArray +var ( + filter_Admin_TSDBSnapshot_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)} +) + func request_Admin_TSDBSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client AdminClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { var protoReq TSDBSnapshotRequest var metadata runtime.ServerMetadata + if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_Admin_TSDBSnapshot_0); err != nil { + return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err) + } + msg, err := client.TSDBSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) return msg, metadata, err @@ -98,7 +106,7 @@ func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, client AdminClient) error { mux.Handle("POST", pattern_Admin_TSDBSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(req.Context()) + ctx, cancel := context.WithCancel(ctx) defer cancel() if cn, ok := w.(http.CloseNotifier); ok { go func(done <-chan struct{}, closed <-chan bool) { @@ -127,7 +135,7 @@ func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, clie }) mux.Handle("POST", pattern_Admin_TSDBCleanTombstones_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(req.Context()) + ctx, cancel := context.WithCancel(ctx) defer cancel() if cn, ok := w.(http.CloseNotifier); ok { go func(done <-chan struct{}, closed <-chan bool) { @@ -156,7 +164,7 @@ func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, clie }) mux.Handle("POST", pattern_Admin_DeleteSeries_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { - ctx, cancel := context.WithCancel(req.Context()) + ctx, cancel := context.WithCancel(ctx) defer cancel() if cn, ok := w.(http.CloseNotifier); ok { go func(done <-chan struct{}, closed <-chan bool) { diff --git a/vendor/github.com/prometheus/prometheus/prompb/rpc.proto b/vendor/github.com/prometheus/prometheus/prompb/rpc.proto index a6463fbfb6..0c1a52f5ae 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/rpc.proto +++ b/vendor/github.com/prometheus/prometheus/prompb/rpc.proto @@ -53,6 +53,7 @@ service Admin { message TSDBSnapshotRequest { + bool skip_head = 1; } message TSDBSnapshotResponse { diff --git a/vendor/github.com/prometheus/prometheus/prompb/types.pb.go b/vendor/github.com/prometheus/prometheus/prompb/types.pb.go index 3edff74f92..59a4a4b0c8 100644 --- a/vendor/github.com/prometheus/prometheus/prompb/types.pb.go +++ b/vendor/github.com/prometheus/prometheus/prompb/types.pb.go @@ -8,6 +8,8 @@ import fmt "fmt" import math "math" import _ "github.com/gogo/protobuf/gogoproto" +import binary "encoding/binary" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -189,7 +191,8 @@ func (m *Sample) MarshalTo(dAtA []byte) (int, error) { if m.Value != 0 { dAtA[i] = 0x9 i++ - i = encodeFixed64Types(dAtA, i, uint64(math.Float64bits(float64(m.Value)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.Value)))) + i += 8 } if m.Timestamp != 0 { dAtA[i] = 0x10 @@ -336,24 +339,6 @@ func (m *LabelMatcher) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func encodeFixed64Types(dAtA []byte, offset int, v uint64) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - dAtA[offset+4] = uint8(v >> 32) - dAtA[offset+5] = uint8(v >> 40) - dAtA[offset+6] = uint8(v >> 48) - dAtA[offset+7] = uint8(v >> 56) - return offset + 8 -} -func encodeFixed32Types(dAtA []byte, offset int, v uint32) int { - dAtA[offset] = uint8(v) - dAtA[offset+1] = uint8(v >> 8) - dAtA[offset+2] = uint8(v >> 16) - dAtA[offset+3] = uint8(v >> 24) - return offset + 4 -} func encodeVarintTypes(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -486,15 +471,8 @@ func (m *Sample) Unmarshal(dAtA []byte) error { if (iNdEx + 8) > l { return io.ErrUnexpectedEOF } + v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - v = uint64(dAtA[iNdEx-8]) - v |= uint64(dAtA[iNdEx-7]) << 8 - v |= uint64(dAtA[iNdEx-6]) << 16 - v |= uint64(dAtA[iNdEx-5]) << 24 - v |= uint64(dAtA[iNdEx-4]) << 32 - v |= uint64(dAtA[iNdEx-3]) << 40 - v |= uint64(dAtA[iNdEx-2]) << 48 - v |= uint64(dAtA[iNdEx-1]) << 56 m.Value = float64(math.Float64frombits(v)) case 2: if wireType != 0 { diff --git a/vendor/github.com/prometheus/prometheus/promql/ast.go b/vendor/github.com/prometheus/prometheus/promql/ast.go index 8871982edf..ccfd9cac98 100644 --- a/vendor/github.com/prometheus/prometheus/promql/ast.go +++ b/vendor/github.com/prometheus/prometheus/promql/ast.go @@ -99,7 +99,7 @@ type Expressions []Expr // AggregateExpr represents an aggregation operation on a Vector. type AggregateExpr struct { - Op itemType // The used aggregation operation. + Op ItemType // The used aggregation operation. Expr Expr // The Vector expression over which is aggregated. Param Expr // Parameter used by some aggregators. Grouping []string // The labels by which to group the Vector. @@ -108,7 +108,7 @@ type AggregateExpr struct { // BinaryExpr represents a binary expression between two child expressions. type BinaryExpr struct { - Op itemType // The operation of the expression. + Op ItemType // The operation of the expression. LHS, RHS Expr // The operands on the respective sides of the operator. // The matching behavior for the operation if both operands are Vectors. @@ -156,7 +156,7 @@ type StringLiteral struct { // UnaryExpr represents a unary operation on another expression. // Currently unary operations are only supported for Scalars. type UnaryExpr struct { - Op itemType + Op ItemType Expr Expr } @@ -238,56 +238,58 @@ type VectorMatching struct { } // Visitor allows visiting a Node and its child nodes. The Visit method is -// invoked for each node encountered by Walk. If the result visitor w is not -// nil, Walk visits each of the children of node with the visitor w, followed -// by a call of w.Visit(nil). +// invoked for each node with the path leading to the node provided additionally. +// If the result visitor w is not nil, Walk visits each of the children +// of node with the visitor w, followed by a call of w.Visit(nil, nil). type Visitor interface { - Visit(node Node) (w Visitor) + Visit(node Node, path []Node) (w Visitor) } // Walk traverses an AST in depth-first order: It starts by calling -// v.Visit(node); node must not be nil. If the visitor w returned by -// v.Visit(node) is not nil, Walk is invoked recursively with visitor +// v.Visit(node, path); node must not be nil. If the visitor w returned by +// v.Visit(node, path) is not nil, Walk is invoked recursively with visitor // w for each of the non-nil children of node, followed by a call of // w.Visit(nil). -func Walk(v Visitor, node Node) { - if v = v.Visit(node); v == nil { +// As the tree is descended the path of previous nodes is provided. +func Walk(v Visitor, node Node, path []Node) { + if v = v.Visit(node, path); v == nil { return } + path = append(path, node) switch n := node.(type) { case Statements: for _, s := range n { - Walk(v, s) + Walk(v, s, path) } case *AlertStmt: - Walk(v, n.Expr) + Walk(v, n.Expr, path) case *EvalStmt: - Walk(v, n.Expr) + Walk(v, n.Expr, path) case *RecordStmt: - Walk(v, n.Expr) + Walk(v, n.Expr, path) case Expressions: for _, e := range n { - Walk(v, e) + Walk(v, e, path) } case *AggregateExpr: - Walk(v, n.Expr) + Walk(v, n.Expr, path) case *BinaryExpr: - Walk(v, n.LHS) - Walk(v, n.RHS) + Walk(v, n.LHS, path) + Walk(v, n.RHS, path) case *Call: - Walk(v, n.Args) + Walk(v, n.Args, path) case *ParenExpr: - Walk(v, n.Expr) + Walk(v, n.Expr, path) case *UnaryExpr: - Walk(v, n.Expr) + Walk(v, n.Expr, path) case *MatrixSelector, *NumberLiteral, *StringLiteral, *VectorSelector: // nothing to do @@ -296,21 +298,21 @@ func Walk(v Visitor, node Node) { panic(fmt.Errorf("promql.Walk: unhandled node type %T", node)) } - v.Visit(nil) + v.Visit(nil, nil) } -type inspector func(Node) bool +type inspector func(Node, []Node) bool -func (f inspector) Visit(node Node) Visitor { - if f(node) { +func (f inspector) Visit(node Node, path []Node) Visitor { + if f(node, path) { return f } return nil } // Inspect traverses an AST in depth-first order: It starts by calling -// f(node); node must not be nil. If f returns true, Inspect invokes f +// f(node, path); node must not be nil. If f returns true, Inspect invokes f // for all the non-nil children of node, recursively. -func Inspect(node Node, f func(Node) bool) { - Walk(inspector(f), node) +func Inspect(node Node, f func(Node, []Node) bool) { + Walk(inspector(f), node, nil) } diff --git a/vendor/github.com/prometheus/prometheus/promql/engine.go b/vendor/github.com/prometheus/prometheus/promql/engine.go index a4503f45e3..8cdbb97d37 100644 --- a/vendor/github.com/prometheus/prometheus/promql/engine.go +++ b/vendor/github.com/prometheus/prometheus/promql/engine.go @@ -89,6 +89,8 @@ type Query interface { // query implements the Query interface. type query struct { + // Underlying data provider. + queryable storage.Queryable // The original query string. q string // Statement of the parsed query. @@ -150,26 +152,18 @@ func contextDone(ctx context.Context, env string) error { // Engine handles the lifetime of queries from beginning to end. // It is connected to a querier. type Engine struct { - // A Querier constructor against an underlying storage. - queryable Queryable - metrics *engineMetrics - // The gate limiting the maximum number of concurrent and waiting queries. + logger log.Logger + metrics *engineMetrics + timeout time.Duration gate *queryGate - options *EngineOptions - - logger log.Logger -} - -// Queryable allows opening a storage querier. -type Queryable interface { - Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) } // NewEngine returns a new engine. -func NewEngine(queryable Queryable, o *EngineOptions) *Engine { - if o == nil { - o = DefaultEngineOptions +func NewEngine(logger log.Logger, reg prometheus.Registerer, maxConcurrent int, timeout time.Duration) *Engine { + if logger == nil { + logger = log.NewNopLogger() } + metrics := &engineMetrics{ currentQueries: prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, @@ -212,10 +206,10 @@ func NewEngine(queryable Queryable, o *EngineOptions) *Engine { ConstLabels: prometheus.Labels{"slice": "result_sort"}, }), } - metrics.maxConcurrentQueries.Set(float64(o.MaxConcurrentQueries)) + metrics.maxConcurrentQueries.Set(float64(maxConcurrent)) - if o.Metrics != nil { - o.Metrics.MustRegister( + if reg != nil { + reg.MustRegister( metrics.currentQueries, metrics.maxConcurrentQueries, metrics.queryInnerEval, @@ -225,36 +219,20 @@ func NewEngine(queryable Queryable, o *EngineOptions) *Engine { ) } return &Engine{ - queryable: queryable, - gate: newQueryGate(o.MaxConcurrentQueries), - options: o, - logger: o.Logger, - metrics: metrics, + gate: newQueryGate(maxConcurrent), + timeout: timeout, + logger: logger, + metrics: metrics, } } -// EngineOptions contains configuration parameters for an Engine. -type EngineOptions struct { - MaxConcurrentQueries int - Timeout time.Duration - Logger log.Logger - Metrics prometheus.Registerer -} - -// DefaultEngineOptions are the default engine options. -var DefaultEngineOptions = &EngineOptions{ - MaxConcurrentQueries: 20, - Timeout: 2 * time.Minute, - Logger: log.NewNopLogger(), -} - // NewInstantQuery returns an evaluation query for the given expression at the given time. -func (ng *Engine) NewInstantQuery(qs string, ts time.Time) (Query, error) { +func (ng *Engine) NewInstantQuery(q storage.Queryable, qs string, ts time.Time) (Query, error) { expr, err := ParseExpr(qs) if err != nil { return nil, err } - qry := ng.newQuery(expr, ts, ts, 0) + qry := ng.newQuery(q, expr, ts, ts, 0) qry.q = qs return qry, nil @@ -262,7 +240,7 @@ func (ng *Engine) NewInstantQuery(qs string, ts time.Time) (Query, error) { // NewRangeQuery returns an evaluation query for the given time range and with // the resolution set by the interval. -func (ng *Engine) NewRangeQuery(qs string, start, end time.Time, interval time.Duration) (Query, error) { +func (ng *Engine) NewRangeQuery(q storage.Queryable, qs string, start, end time.Time, interval time.Duration) (Query, error) { expr, err := ParseExpr(qs) if err != nil { return nil, err @@ -270,13 +248,13 @@ func (ng *Engine) NewRangeQuery(qs string, start, end time.Time, interval time.D if expr.Type() != ValueTypeVector && expr.Type() != ValueTypeScalar { return nil, fmt.Errorf("invalid expression type %q for range query, must be Scalar or instant Vector", documentedType(expr.Type())) } - qry := ng.newQuery(expr, start, end, interval) + qry := ng.newQuery(q, expr, start, end, interval) qry.q = qs return qry, nil } -func (ng *Engine) newQuery(expr Expr, start, end time.Time, interval time.Duration) *query { +func (ng *Engine) newQuery(q storage.Queryable, expr Expr, start, end time.Time, interval time.Duration) *query { es := &EvalStmt{ Expr: expr, Start: start, @@ -284,9 +262,10 @@ func (ng *Engine) newQuery(expr Expr, start, end time.Time, interval time.Durati Interval: interval, } qry := &query{ - stmt: es, - ng: ng, - stats: stats.NewTimerGroup(), + stmt: es, + ng: ng, + stats: stats.NewTimerGroup(), + queryable: q, } return qry } @@ -316,7 +295,7 @@ func (ng *Engine) exec(ctx context.Context, q *query) (Value, error) { ng.metrics.currentQueries.Inc() defer ng.metrics.currentQueries.Dec() - ctx, cancel := context.WithTimeout(ctx, ng.options.Timeout) + ctx, cancel := context.WithTimeout(ctx, ng.timeout) q.cancel = cancel execTimer := q.stats.GetTimer(stats.ExecTotalTime).Start() @@ -363,9 +342,8 @@ func durationMilliseconds(d time.Duration) int64 { // execEvalStmt evaluates the expression of an evaluation statement for the given time range. func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) (Value, error) { - prepareTimer := query.stats.GetTimer(stats.QueryPreparationTime).Start() - querier, err := ng.populateIterators(ctx, s) + querier, err := ng.populateIterators(ctx, query.queryable, s) prepareTimer.Stop() ng.metrics.queryPrepareTime.Observe(prepareTimer.ElapsedTime().Seconds()) @@ -489,10 +467,9 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *EvalStmt) ( return mat, nil } -func (ng *Engine) populateIterators(ctx context.Context, s *EvalStmt) (storage.Querier, error) { +func (ng *Engine) populateIterators(ctx context.Context, q storage.Queryable, s *EvalStmt) (storage.Querier, error) { var maxOffset time.Duration - - Inspect(s.Expr, func(node Node) bool { + Inspect(s.Expr, func(node Node, _ []Node) bool { switch n := node.(type) { case *VectorSelector: if maxOffset < LookbackDelta { @@ -514,15 +491,22 @@ func (ng *Engine) populateIterators(ctx context.Context, s *EvalStmt) (storage.Q mint := s.Start.Add(-maxOffset) - querier, err := ng.queryable.Querier(ctx, timestamp.FromTime(mint), timestamp.FromTime(s.End)) + querier, err := q.Querier(ctx, timestamp.FromTime(mint), timestamp.FromTime(s.End)) if err != nil { return nil, err } - Inspect(s.Expr, func(node Node) bool { + Inspect(s.Expr, func(node Node, path []Node) bool { + var set storage.SeriesSet + params := &storage.SelectParams{ + Step: int64(s.Interval / time.Millisecond), + } + switch n := node.(type) { case *VectorSelector: - set, err := querier.Select(n.LabelMatchers...) + params.Func = extractFuncFromPath(path) + + set, err = querier.Select(params, n.LabelMatchers...) if err != nil { level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) return false @@ -539,7 +523,9 @@ func (ng *Engine) populateIterators(ctx context.Context, s *EvalStmt) (storage.Q } case *MatrixSelector: - set, err := querier.Select(n.LabelMatchers...) + params.Func = extractFuncFromPath(path) + + set, err = querier.Select(params, n.LabelMatchers...) if err != nil { level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) return false @@ -559,6 +545,25 @@ func (ng *Engine) populateIterators(ctx context.Context, s *EvalStmt) (storage.Q return querier, err } +// extractFuncFromPath walks up the path and searches for the first instance of +// a function or aggregation. +func extractFuncFromPath(p []Node) string { + if len(p) == 0 { + return "" + } + switch n := p[len(p)-1].(type) { + case *AggregateExpr: + return n.Op.String() + case *Call: + return n.Func.Name + case *BinaryExpr: + // If we hit a binary expression we terminate since we only care about functions + // or aggregations over a single metric. + return "" + } + return extractFuncFromPath(p[:len(p)-1]) +} + func expandSeriesSet(it storage.SeriesSet) (res []storage.Series, err error) { for it.Next() { res = append(res, it.At()) @@ -957,7 +962,7 @@ func (ev *evaluator) VectorUnless(lhs, rhs Vector, matching *VectorMatching) Vec } // VectorBinop evaluates a binary operation between two Vectors, excluding set operators. -func (ev *evaluator) VectorBinop(op itemType, lhs, rhs Vector, matching *VectorMatching, returnBool bool) Vector { +func (ev *evaluator) VectorBinop(op ItemType, lhs, rhs Vector, matching *VectorMatching, returnBool bool) Vector { if matching.Card == CardManyToMany { panic("many-to-many only allowed for set operators") } @@ -1095,7 +1100,7 @@ func signatureFunc(on bool, names ...string) func(labels.Labels) uint64 { // resultMetric returns the metric for the given sample(s) based on the Vector // binary operation and the matching options. -func resultMetric(lhs, rhs labels.Labels, op itemType, matching *VectorMatching) labels.Labels { +func resultMetric(lhs, rhs labels.Labels, op ItemType, matching *VectorMatching) labels.Labels { lb := labels.NewBuilder(lhs) if shouldDropMetricName(op) { @@ -1130,7 +1135,7 @@ func resultMetric(lhs, rhs labels.Labels, op itemType, matching *VectorMatching) } // VectorscalarBinop evaluates a binary operation between a Vector and a Scalar. -func (ev *evaluator) VectorscalarBinop(op itemType, lhs Vector, rhs Scalar, swap, returnBool bool) Vector { +func (ev *evaluator) VectorscalarBinop(op ItemType, lhs Vector, rhs Scalar, swap, returnBool bool) Vector { vec := make(Vector, 0, len(lhs)) for _, lhsSample := range lhs { @@ -1151,7 +1156,7 @@ func (ev *evaluator) VectorscalarBinop(op itemType, lhs Vector, rhs Scalar, swap } if keep { lhsSample.V = value - if shouldDropMetricName(op) { + if shouldDropMetricName(op) || returnBool { lhsSample.Metric = dropMetricName(lhsSample.Metric) } vec = append(vec, lhsSample) @@ -1165,7 +1170,7 @@ func dropMetricName(l labels.Labels) labels.Labels { } // scalarBinop evaluates a binary operation between two Scalars. -func scalarBinop(op itemType, lhs, rhs float64) float64 { +func scalarBinop(op ItemType, lhs, rhs float64) float64 { switch op { case itemADD: return lhs + rhs @@ -1196,7 +1201,7 @@ func scalarBinop(op itemType, lhs, rhs float64) float64 { } // vectorElemBinop evaluates a binary operation between two Vector elements. -func vectorElemBinop(op itemType, lhs, rhs float64) (float64, bool) { +func vectorElemBinop(op ItemType, lhs, rhs float64) (float64, bool) { switch op { case itemADD: return lhs + rhs, true @@ -1251,7 +1256,7 @@ type groupedAggregation struct { } // aggregation evaluates an aggregation operation on a Vector. -func (ev *evaluator) aggregation(op itemType, grouping []string, without bool, param Expr, vec Vector) Vector { +func (ev *evaluator) aggregation(op ItemType, grouping []string, without bool, param Expr, vec Vector) Vector { result := map[uint64]*groupedAggregation{} var k int64 @@ -1457,7 +1462,7 @@ func btos(b bool) float64 { // shouldDropMetricName returns whether the metric name should be dropped in the // result of the op operation. -func shouldDropMetricName(op itemType) bool { +func shouldDropMetricName(op ItemType) bool { switch op { case itemADD, itemSUB, itemDIV, itemMUL, itemMOD: return true diff --git a/vendor/github.com/prometheus/prometheus/promql/lex.go b/vendor/github.com/prometheus/prometheus/promql/lex.go index 8a4d83b052..bb2476cf69 100644 --- a/vendor/github.com/prometheus/prometheus/promql/lex.go +++ b/vendor/github.com/prometheus/prometheus/promql/lex.go @@ -22,7 +22,7 @@ import ( // item represents a token or text string returned from the scanner. type item struct { - typ itemType // The type of this item. + typ ItemType // The type of this item. pos Pos // The starting position, in bytes, of this item in the input string. val string // The value of this item. } @@ -50,25 +50,25 @@ func (i item) String() string { // isOperator returns true if the item corresponds to a arithmetic or set operator. // Returns false otherwise. -func (i itemType) isOperator() bool { return i > operatorsStart && i < operatorsEnd } +func (i ItemType) isOperator() bool { return i > operatorsStart && i < operatorsEnd } // isAggregator returns true if the item belongs to the aggregator functions. // Returns false otherwise -func (i itemType) isAggregator() bool { return i > aggregatorsStart && i < aggregatorsEnd } +func (i ItemType) isAggregator() bool { return i > aggregatorsStart && i < aggregatorsEnd } // isAggregator returns true if the item is an aggregator that takes a parameter. // Returns false otherwise -func (i itemType) isAggregatorWithParam() bool { +func (i ItemType) isAggregatorWithParam() bool { return i == itemTopK || i == itemBottomK || i == itemCountValues || i == itemQuantile } // isKeyword returns true if the item corresponds to a keyword. // Returns false otherwise. -func (i itemType) isKeyword() bool { return i > keywordsStart && i < keywordsEnd } +func (i ItemType) isKeyword() bool { return i > keywordsStart && i < keywordsEnd } // isCompairsonOperator returns true if the item corresponds to a comparison operator. // Returns false otherwise. -func (i itemType) isComparisonOperator() bool { +func (i ItemType) isComparisonOperator() bool { switch i { case itemEQL, itemNEQ, itemLTE, itemLSS, itemGTE, itemGTR: return true @@ -78,7 +78,7 @@ func (i itemType) isComparisonOperator() bool { } // isSetOperator returns whether the item corresponds to a set operator. -func (i itemType) isSetOperator() bool { +func (i ItemType) isSetOperator() bool { switch i { case itemLAND, itemLOR, itemLUnless: return true @@ -92,7 +92,7 @@ const LowestPrec = 0 // Non-operators. // Precedence returns the operator precedence of the binary // operator op. If op is not a binary operator, the result // is LowestPrec. -func (i itemType) precedence() int { +func (i ItemType) precedence() int { switch i { case itemLOR: return 1 @@ -111,7 +111,7 @@ func (i itemType) precedence() int { } } -func (i itemType) isRightAssociative() bool { +func (i ItemType) isRightAssociative() bool { switch i { case itemPOW: return true @@ -121,10 +121,10 @@ func (i itemType) isRightAssociative() bool { } -type itemType int +type ItemType int const ( - itemError itemType = iota // Error occurred, value is error message + itemError ItemType = iota // Error occurred, value is error message itemEOF itemComment itemIdentifier @@ -198,7 +198,7 @@ const ( keywordsEnd ) -var key = map[string]itemType{ +var key = map[string]ItemType{ // Operators. "and": itemLAND, "or": itemLOR, @@ -235,7 +235,7 @@ var key = map[string]itemType{ // These are the default string representations for common items. It does not // imply that those are the only character sequences that can be lexed to such an item. -var itemTypeStr = map[itemType]string{ +var itemTypeStr = map[ItemType]string{ itemLeftParen: "(", itemRightParen: ")", itemLeftBrace: "{", @@ -274,7 +274,7 @@ func init() { key["nan"] = itemNumber } -func (i itemType) String() string { +func (i ItemType) String() string { if s, ok := itemTypeStr[i]; ok { return s } @@ -291,7 +291,7 @@ func (i item) desc() string { return fmt.Sprintf("%s %s", i.typ.desc(), i) } -func (i itemType) desc() string { +func (i ItemType) desc() string { switch i { case itemError: return "error" @@ -366,7 +366,7 @@ func (l *lexer) backup() { } // emit passes an item back to the client. -func (l *lexer) emit(t itemType) { +func (l *lexer) emit(t ItemType) { l.items <- item{t, l.start, l.input[l.start:l.pos]} l.start = l.pos } diff --git a/vendor/github.com/prometheus/prometheus/promql/parse.go b/vendor/github.com/prometheus/prometheus/promql/parse.go index 7008b148a9..79299a4c20 100644 --- a/vendor/github.com/prometheus/prometheus/promql/parse.go +++ b/vendor/github.com/prometheus/prometheus/promql/parse.go @@ -104,14 +104,6 @@ func ParseMetricSelector(input string) (m []*labels.Matcher, err error) { return vs.LabelMatchers, nil } -// parseSeriesDesc parses the description of a time series. -func parseSeriesDesc(input string) (labels.Labels, []sequenceValue, error) { - p := newParser(input) - p.lex.seriesDesc = true - - return p.parseSeriesDesc() -} - // newParser returns a new parser. func newParser(input string) *parser { p := &parser{ @@ -167,6 +159,14 @@ func (v sequenceValue) String() string { return fmt.Sprintf("%f", v.value) } +// parseSeriesDesc parses the description of a time series. +func parseSeriesDesc(input string) (labels.Labels, []sequenceValue, error) { + p := newParser(input) + p.lex.seriesDesc = true + + return p.parseSeriesDesc() +} + // parseSeriesDesc parses a description of a time series into its metric and value sequence. func (p *parser) parseSeriesDesc() (m labels.Labels, vals []sequenceValue, err error) { defer p.recover(&err) @@ -314,7 +314,7 @@ func (p *parser) error(err error) { } // expect consumes the next token and guarantees it has the required type. -func (p *parser) expect(exp itemType, context string) item { +func (p *parser) expect(exp ItemType, context string) item { token := p.next() if token.typ != exp { p.errorf("unexpected %s in %s, expected %s", token.desc(), context, exp.desc()) @@ -323,7 +323,7 @@ func (p *parser) expect(exp itemType, context string) item { } // expectOneOf consumes the next token and guarantees it has one of the required types. -func (p *parser) expectOneOf(exp1, exp2 itemType, context string) item { +func (p *parser) expectOneOf(exp1, exp2 ItemType, context string) item { token := p.next() if token.typ != exp1 && token.typ != exp2 { p.errorf("unexpected %s in %s, expected %s or %s", token.desc(), context, exp1.desc(), exp2.desc()) @@ -348,7 +348,6 @@ func (p *parser) recover(errp *error) { *errp = e.(error) } } - return } // stmt parses any statement. @@ -510,7 +509,7 @@ func (p *parser) expr() Expr { } } -func (p *parser) balance(lhs Expr, op itemType, rhs Expr, vecMatching *VectorMatching, returnBool bool) *BinaryExpr { +func (p *parser) balance(lhs Expr, op ItemType, rhs Expr, vecMatching *VectorMatching, returnBool bool) *BinaryExpr { if lhsBE, ok := lhs.(*BinaryExpr); ok { precd := lhsBE.Op.precedence() - op.precedence() if (precd < 0) || (precd == 0 && op.isRightAssociative()) { @@ -811,7 +810,7 @@ func (p *parser) labelSet() labels.Labels { // // '{' [ , ... ] '}' // -func (p *parser) labelMatchers(operators ...itemType) []*labels.Matcher { +func (p *parser) labelMatchers(operators ...ItemType) []*labels.Matcher { const ctx = "label matching" matchers := []*labels.Matcher{} diff --git a/vendor/github.com/prometheus/prometheus/promql/test.go b/vendor/github.com/prometheus/prometheus/promql/test.go index b600b00f41..6bd07cf7ec 100644 --- a/vendor/github.com/prometheus/prometheus/promql/test.go +++ b/vendor/github.com/prometheus/prometheus/promql/test.go @@ -83,6 +83,11 @@ func (t *Test) QueryEngine() *Engine { return t.queryEngine } +// Queryable allows querying the test data. +func (t *Test) Queryable() storage.Queryable { + return t.storage +} + // Context returns the test's context. func (t *Test) Context() context.Context { return t.context @@ -136,15 +141,15 @@ func (t *Test) parseEval(lines []string, i int) (int, *evalCmd, error) { } parts := patEvalInstant.FindStringSubmatch(lines[i]) var ( - mod = parts[1] - at = parts[2] - qry = parts[3] + mod = parts[1] + at = parts[2] + expr = parts[3] ) - expr, err := ParseExpr(qry) + _, err := ParseExpr(expr) if err != nil { if perr, ok := err.(*ParseErr); ok { perr.Line = i + 1 - perr.Pos += strings.Index(lines[i], qry) + perr.Pos += strings.Index(lines[i], expr) } return i, nil, err } @@ -296,7 +301,7 @@ func (cmd *loadCmd) append(a storage.Appender) error { // evalCmd is a command that evaluates an expression for the given time (range) // and expects a specific result. type evalCmd struct { - expr Expr + expr string start, end time.Time interval time.Duration @@ -316,7 +321,7 @@ func (e entry) String() string { return fmt.Sprintf("%d: %s", e.pos, e.vals) } -func newEvalCmd(expr Expr, start, end time.Time, interval time.Duration) *evalCmd { +func newEvalCmd(expr string, start, end time.Time, interval time.Duration) *evalCmd { return &evalCmd{ expr: expr, start: start, @@ -380,11 +385,6 @@ func (ev *evalCmd) compareResult(result Value) error { return fmt.Errorf("received instant result on range evaluation") } - fmt.Println("vector result", len(val), ev.expr) - for _, ss := range val { - fmt.Println(" ", ss.Metric, ss.Point) - } - seen := map[uint64]bool{} for pos, v := range val { fp := v.Metric.Hash() @@ -403,6 +403,10 @@ func (ev *evalCmd) compareResult(result Value) error { } for fp, expVals := range ev.expected { if !seen[fp] { + fmt.Println("vector result", len(val), ev.expr) + for _, ss := range val { + fmt.Println(" ", ss.Metric, ss.Point) + } return fmt.Errorf("expected metric %s with %v not found", ev.metrics[fp], expVals) } } @@ -460,7 +464,8 @@ func (t *Test) exec(tc testCommand) error { } case *evalCmd: - q := t.queryEngine.newQuery(cmd.expr, cmd.start, cmd.end, cmd.interval) + qry, _ := ParseExpr(cmd.expr) + q := t.queryEngine.newQuery(t.storage, qry, cmd.start, cmd.end, cmd.interval) res := q.Exec(t.context) if res.Err != nil { if cmd.fail { @@ -495,7 +500,7 @@ func (t *Test) clear() { } t.storage = testutil.NewStorage(t) - t.queryEngine = NewEngine(t.storage, nil) + t.queryEngine = NewEngine(nil, nil, 20, 10*time.Second) t.context, t.cancelCtx = context.WithCancel(context.Background()) } diff --git a/vendor/github.com/prometheus/prometheus/rules/alerting.go b/vendor/github.com/prometheus/prometheus/rules/alerting.go index 94f266dd9f..e927d399f9 100644 --- a/vendor/github.com/prometheus/prometheus/rules/alerting.go +++ b/vendor/github.com/prometheus/prometheus/rules/alerting.go @@ -207,7 +207,7 @@ func (r *AlertingRule) Eval(ctx context.Context, ts time.Time, query QueryFunc, expand := func(text string) string { tmpl := template.NewTemplateExpander( ctx, - defs+string(text), + defs+text, "__alert_"+r.Name(), tmplData, model.Time(timestamp.FromTime(ts)), diff --git a/vendor/github.com/prometheus/prometheus/rules/manager.go b/vendor/github.com/prometheus/prometheus/rules/manager.go index 4328e5c15c..b2a8a51c88 100644 --- a/vendor/github.com/prometheus/prometheus/rules/manager.go +++ b/vendor/github.com/prometheus/prometheus/rules/manager.go @@ -106,9 +106,9 @@ type QueryFunc func(ctx context.Context, q string, t time.Time) (promql.Vector, // EngineQueryFunc returns a new query function that executes instant queries against // the given engine. // It converts scaler into vector results. -func EngineQueryFunc(engine *promql.Engine) QueryFunc { +func EngineQueryFunc(engine *promql.Engine, q storage.Queryable) QueryFunc { return func(ctx context.Context, qs string, t time.Time) (promql.Vector, error) { - q, err := engine.NewInstantQuery(qs, t) + q, err := engine.NewInstantQuery(q, qs, t) if err != nil { return nil, err } @@ -286,12 +286,12 @@ func (g *Group) copyState(from *Group) { ruleMap := make(map[string][]int, len(from.rules)) for fi, fromRule := range from.rules { - l, _ := ruleMap[fromRule.Name()] + l := ruleMap[fromRule.Name()] ruleMap[fromRule.Name()] = append(l, fi) } for i, rule := range g.rules { - indexes, _ := ruleMap[rule.Name()] + indexes := ruleMap[rule.Name()] if len(indexes) == 0 { continue } diff --git a/vendor/github.com/prometheus/prometheus/scrape/manager.go b/vendor/github.com/prometheus/prometheus/scrape/manager.go index b498ea0b40..64fc100b0b 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/manager.go +++ b/vendor/github.com/prometheus/prometheus/scrape/manager.go @@ -132,6 +132,19 @@ func (m *Manager) Targets() []*Target { return targets } +// DroppedTargets returns the targets dropped during relabelling. +func (m *Manager) DroppedTargets() []*Target { + m.mtx.Lock() + defer m.mtx.Unlock() + var droppedTargets []*Target + for _, p := range m.scrapePools { + p.mtx.RLock() + droppedTargets = append(droppedTargets, p.droppedTargets...) + p.mtx.RUnlock() + } + return droppedTargets +} + func (m *Manager) reload(t map[string][]*targetgroup.Group) { for tsetName, tgroup := range t { scrapeConfig, ok := m.scrapeConfigs[tsetName] diff --git a/vendor/github.com/prometheus/prometheus/scrape/scrape.go b/vendor/github.com/prometheus/prometheus/scrape/scrape.go index a22dcb9a36..1c3adfb721 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/scrape.go +++ b/vendor/github.com/prometheus/prometheus/scrape/scrape.go @@ -148,7 +148,7 @@ func newScrapePool(cfg *config.ScrapeConfig, app Appendable, logger log.Logger) level.Error(logger).Log("msg", "Error creating HTTP client", "err", err) } - buffers := pool.NewBytesPool(163, 100e6, 3) + buffers := pool.New(163, 100e6, 3, func(sz int) interface{} { return make([]byte, 0, sz) }) ctx, cancel := context.WithCancel(context.Background()) sp := &scrapePool{ @@ -336,7 +336,7 @@ func (sp *scrapePool) mutateSampleLabels(lset labels.Labels, target *Target) lab if sp.config.HonorLabels { for _, l := range target.Labels() { - if lv := lset.Get(l.Name); lv == "" { + if !lset.Has(l.Name) { lb.Set(l.Name, l.Value) } } @@ -350,6 +350,12 @@ func (sp *scrapePool) mutateSampleLabels(lset labels.Labels, target *Target) lab } } + for _, l := range lb.Labels() { + if l.Value == "" { + lb.Del(l.Name) + } + } + res := lb.Labels() if mrc := sp.config.MetricRelabelConfigs; len(mrc) > 0 { @@ -481,7 +487,7 @@ type scrapeLoop struct { l log.Logger cache *scrapeCache lastScrapeSize int - buffers *pool.BytesPool + buffers *pool.Pool appender func() storage.Appender sampleMutator labelsMutator @@ -596,7 +602,7 @@ func (c *scrapeCache) forEachStale(f func(labels.Labels) bool) { func newScrapeLoop(ctx context.Context, sc scraper, l log.Logger, - buffers *pool.BytesPool, + buffers *pool.Pool, sampleMutator labelsMutator, reportSampleMutator labelsMutator, appender func() storage.Appender, @@ -605,7 +611,7 @@ func newScrapeLoop(ctx context.Context, l = log.NewNopLogger() } if buffers == nil { - buffers = pool.NewBytesPool(1e3, 1e6, 3) + buffers = pool.New(1e3, 1e6, 3, func(sz int) interface{} { return make([]byte, 0, sz) }) } sl := &scrapeLoop{ scraper: sc, @@ -662,7 +668,8 @@ mainLoop: time.Since(last).Seconds(), ) } - b := sl.buffers.Get(sl.lastScrapeSize) + + b := sl.buffers.Get(sl.lastScrapeSize).([]byte) buf := bytes.NewBuffer(b) scrapeErr := sl.scraper.scrape(scrapeCtx, buf) diff --git a/vendor/github.com/prometheus/prometheus/scrape/target.go b/vendor/github.com/prometheus/prometheus/scrape/target.go index 6241d52281..3ead167327 100644 --- a/vendor/github.com/prometheus/prometheus/scrape/target.go +++ b/vendor/github.com/prometheus/prometheus/scrape/target.go @@ -135,16 +135,16 @@ func (t *Target) URL() *url.URL { ks := l.Name[len(model.ParamLabelPrefix):] if len(params[ks]) > 0 { - params[ks][0] = string(l.Value) + params[ks][0] = l.Value } else { params[ks] = []string{l.Value} } } return &url.URL{ - Scheme: string(t.labels.Get(model.SchemeLabel)), - Host: string(t.labels.Get(model.AddressLabel)), - Path: string(t.labels.Get(model.MetricsPathLabel)), + Scheme: t.labels.Get(model.SchemeLabel), + Host: t.labels.Get(model.AddressLabel), + Path: t.labels.Get(model.MetricsPathLabel), RawQuery: params.Encode(), } } diff --git a/vendor/github.com/prometheus/prometheus/storage/buffer.go b/vendor/github.com/prometheus/prometheus/storage/buffer.go index 0159cfa3fb..77476c6146 100644 --- a/vendor/github.com/prometheus/prometheus/storage/buffer.go +++ b/vendor/github.com/prometheus/prometheus/storage/buffer.go @@ -178,6 +178,7 @@ func (r *sampleRing) add(t int64, v float64) { r.buf = buf r.i = r.f r.f += l + l = 2 * l } else { r.i++ if r.i >= l { diff --git a/vendor/github.com/prometheus/prometheus/storage/fanout.go b/vendor/github.com/prometheus/prometheus/storage/fanout.go index 40be2536d6..7835d1ef7c 100644 --- a/vendor/github.com/prometheus/prometheus/storage/fanout.go +++ b/vendor/github.com/prometheus/prometheus/storage/fanout.go @@ -216,10 +216,10 @@ func NewMergeQuerier(queriers []Querier) Querier { } // Select returns a set of series that matches the given label matchers. -func (q *mergeQuerier) Select(matchers ...*labels.Matcher) (SeriesSet, error) { +func (q *mergeQuerier) Select(params *SelectParams, matchers ...*labels.Matcher) (SeriesSet, error) { seriesSets := make([]SeriesSet, 0, len(q.queriers)) for _, querier := range q.queriers { - set, err := querier.Select(matchers...) + set, err := querier.Select(params, matchers...) if err != nil { return nil, err } diff --git a/vendor/github.com/prometheus/prometheus/storage/interface.go b/vendor/github.com/prometheus/prometheus/storage/interface.go index e04126cc9c..9c145c4259 100644 --- a/vendor/github.com/prometheus/prometheus/storage/interface.go +++ b/vendor/github.com/prometheus/prometheus/storage/interface.go @@ -52,7 +52,7 @@ type Queryable interface { // Querier provides reading access to time series data. type Querier interface { // Select returns a set of series that matches the given label matchers. - Select(...*labels.Matcher) (SeriesSet, error) + Select(*SelectParams, ...*labels.Matcher) (SeriesSet, error) // LabelValues returns all potential values for a label name. LabelValues(name string) ([]string, error) @@ -61,6 +61,12 @@ type Querier interface { Close() error } +// SelectParams specifies parameters passed to data selections. +type SelectParams struct { + Step int64 // Query step size in milliseconds. + Func string // String representation of surrounding function or aggregation. +} + // QueryableFunc is an adapter to allow the use of ordinary functions as // Queryables. It follows the idea of http.HandlerFunc. type QueryableFunc func(ctx context.Context, mint, maxt int64) (Querier, error) diff --git a/vendor/github.com/prometheus/prometheus/storage/noop.go b/vendor/github.com/prometheus/prometheus/storage/noop.go index a5ff1bc9b4..fa024430f5 100644 --- a/vendor/github.com/prometheus/prometheus/storage/noop.go +++ b/vendor/github.com/prometheus/prometheus/storage/noop.go @@ -22,7 +22,7 @@ func NoopQuerier() Querier { return noopQuerier{} } -func (noopQuerier) Select(...*labels.Matcher) (SeriesSet, error) { +func (noopQuerier) Select(*SelectParams, ...*labels.Matcher) (SeriesSet, error) { return NoopSeriesSet(), nil } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go b/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go index 9c7f66a94c..7d52ed1eff 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/queue_manager.go @@ -196,8 +196,7 @@ func NewQueueManager(logger log.Logger, cfg config.QueueConfig, externalLabels m // sample on the floor if the queue is full. // Always returns nil. func (t *QueueManager) Append(s *model.Sample) error { - var snew model.Sample - snew = *s + snew := *s snew.Metric = s.Metric.Clone() for ln, lv := range t.externalLabels { @@ -431,6 +430,15 @@ func (s *shards) runShard(i int) { pendingSamples := model.Samples{} timer := time.NewTimer(s.qm.cfg.BatchSendDeadline) + stop := func() { + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + } + defer stop() for { select { @@ -447,19 +455,20 @@ func (s *shards) runShard(i int) { queueLength.WithLabelValues(s.qm.queueName).Dec() pendingSamples = append(pendingSamples, sample) - for len(pendingSamples) >= s.qm.cfg.MaxSamplesPerSend { + if len(pendingSamples) >= s.qm.cfg.MaxSamplesPerSend { s.sendSamples(pendingSamples[:s.qm.cfg.MaxSamplesPerSend]) pendingSamples = pendingSamples[s.qm.cfg.MaxSamplesPerSend:] + + stop() + timer.Reset(s.qm.cfg.BatchSendDeadline) } - if !timer.Stop() { - <-timer.C - } - timer.Reset(s.qm.cfg.BatchSendDeadline) + case <-timer.C: if len(pendingSamples) > 0 { s.sendSamples(pendingSamples) pendingSamples = pendingSamples[:0] } + timer.Reset(s.qm.cfg.BatchSendDeadline) } } } diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/read.go b/vendor/github.com/prometheus/prometheus/storage/remote/read.go index e49d3524a5..1fedad8846 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/read.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/read.go @@ -43,7 +43,7 @@ type querier struct { // Select implements storage.Querier and uses the given matchers to read series // sets from the Client. -func (q *querier) Select(matchers ...*labels.Matcher) (storage.SeriesSet, error) { +func (q *querier) Select(_ *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, error) { query, err := ToQuery(q.mint, q.maxt, matchers) if err != nil { return nil, err @@ -91,9 +91,9 @@ type externalLabelsQuerier struct { // Select adds equality matchers for all external labels to the list of matchers // before calling the wrapped storage.Queryable. The added external labels are // removed from the returned series sets. -func (q externalLabelsQuerier) Select(matchers ...*labels.Matcher) (storage.SeriesSet, error) { +func (q externalLabelsQuerier) Select(p *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, error) { m, added := q.addExternalLabels(matchers) - s, err := q.Querier.Select(m...) + s, err := q.Querier.Select(p, m...) if err != nil { return nil, err } @@ -144,7 +144,7 @@ type requiredMatchersQuerier struct { // Select returns a NoopSeriesSet if the given matchers don't match the label // set of the requiredMatchersQuerier. Otherwise it'll call the wrapped querier. -func (q requiredMatchersQuerier) Select(matchers ...*labels.Matcher) (storage.SeriesSet, error) { +func (q requiredMatchersQuerier) Select(p *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, error) { ms := q.requiredMatchers for _, m := range matchers { for i, r := range ms { @@ -160,7 +160,7 @@ func (q requiredMatchersQuerier) Select(matchers ...*labels.Matcher) (storage.Se if len(ms) > 0 { return storage.NoopSeriesSet(), nil } - return q.Querier.Select(matchers...) + return q.Querier.Select(p, matchers...) } // addExternalLabels adds matchers for each external label. External labels diff --git a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go index c4cfd72344..5192126eac 100644 --- a/vendor/github.com/prometheus/prometheus/storage/remote/storage.go +++ b/vendor/github.com/prometheus/prometheus/storage/remote/storage.go @@ -99,8 +99,7 @@ func (s *Storage) ApplyConfig(conf *config.Config) error { return err } - var q storage.Queryable - q = QueryableClient(c) + q := QueryableClient(c) q = ExternablLabelsHandler(q, conf.GlobalConfig.ExternalLabels) if len(rrConf.RequiredMatchers) > 0 { q = RequiredMatchersFilter(q, labelsToEqualityMatchers(rrConf.RequiredMatchers)) diff --git a/vendor/github.com/prometheus/prometheus/storage/tsdb/tsdb.go b/vendor/github.com/prometheus/prometheus/storage/tsdb/tsdb.go index 070488fbf1..92e8cadd3e 100644 --- a/vendor/github.com/prometheus/prometheus/storage/tsdb/tsdb.go +++ b/vendor/github.com/prometheus/prometheus/storage/tsdb/tsdb.go @@ -159,7 +159,7 @@ func (a adapter) StartTime() (int64, error) { if len(a.db.Blocks()) > 0 { startTime = a.db.Blocks()[0].Meta().MinTime } else { - startTime = int64(time.Now().Unix() * 1000) + startTime = time.Now().Unix() * 1000 } // Add a safety margin as it may take a few minutes for everything to spin up. @@ -188,7 +188,7 @@ type querier struct { q tsdb.Querier } -func (q querier) Select(oms ...*labels.Matcher) (storage.SeriesSet, error) { +func (q querier) Select(_ *storage.SelectParams, oms ...*labels.Matcher) (storage.SeriesSet, error) { ms := make([]tsdbLabels.Matcher, 0, len(oms)) for _, om := range oms { diff --git a/vendor/github.com/prometheus/prometheus/template/template.go b/vendor/github.com/prometheus/prometheus/template/template.go index 71bc486e15..91ff9d5eb0 100644 --- a/vendor/github.com/prometheus/prometheus/template/template.go +++ b/vendor/github.com/prometheus/prometheus/template/template.go @@ -265,8 +265,7 @@ func (te Expander) Expand() (result string, resultErr error) { } }() - tmpl, err := text_template.New(te.name).Funcs(te.funcMap).Parse(te.text) - tmpl.Option("missingkey=zero") + tmpl, err := text_template.New(te.name).Funcs(te.funcMap).Option("missingkey=zero").Parse(te.text) if err != nil { return "", fmt.Errorf("error parsing template %v: %v", te.name, err) } diff --git a/vendor/github.com/prometheus/prometheus/util/testutil/error.go b/vendor/github.com/prometheus/prometheus/util/testutil/error.go index a132abf8f4..38fa71adb3 100644 --- a/vendor/github.com/prometheus/prometheus/util/testutil/error.go +++ b/vendor/github.com/prometheus/prometheus/util/testutil/error.go @@ -20,11 +20,7 @@ func ErrorEqual(left, right error) bool { } if left != nil && right != nil { - if left.Error() == right.Error() { - return true - } - - return false + return left.Error() == right.Error() } return false diff --git a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go index 523316c901..dd9cc7267c 100644 --- a/vendor/github.com/prometheus/prometheus/web/api/v1/api.go +++ b/vendor/github.com/prometheus/prometheus/web/api/v1/api.go @@ -15,7 +15,6 @@ package v1 import ( "context" - "encoding/json" "errors" "fmt" "math" @@ -27,8 +26,9 @@ import ( "sort" "strconv" "time" + "unsafe" - "github.com/prometheus/client_golang/prometheus" + jsoniter "github.com/json-iterator/go" "github.com/prometheus/common/model" "github.com/prometheus/common/route" "github.com/prometheus/tsdb" @@ -83,10 +83,12 @@ func (e *apiError) Error() string { type targetRetriever interface { Targets() []*scrape.Target + DroppedTargets() []*scrape.Target } type alertmanagerRetriever interface { Alertmanagers() []*url.URL + DroppedAlertmanagers() []*url.URL } type response struct { @@ -108,15 +110,16 @@ type apiFunc func(r *http.Request) (interface{}, *apiError) // API can register a set of endpoints in a router and handle // them using the provided storage and query engine. type API struct { - Queryable promql.Queryable + Queryable storage.Queryable QueryEngine *promql.Engine targetRetriever targetRetriever alertmanagerRetriever alertmanagerRetriever - now func() time.Time - config func() config.Config - ready func(http.HandlerFunc) http.HandlerFunc + now func() time.Time + config func() config.Config + flagsMap map[string]string + ready func(http.HandlerFunc) http.HandlerFunc db func() *tsdb.DB enableAdmin bool @@ -125,10 +128,11 @@ type API struct { // NewAPI returns an initialized API type. func NewAPI( qe *promql.Engine, - q promql.Queryable, + q storage.Queryable, tr targetRetriever, ar alertmanagerRetriever, configFunc func() config.Config, + flagsMap map[string]string, readyFunc func(http.HandlerFunc) http.HandlerFunc, db func() *tsdb.DB, enableAdmin bool, @@ -140,6 +144,7 @@ func NewAPI( alertmanagerRetriever: ar, now: time.Now, config: configFunc, + flagsMap: flagsMap, ready: readyFunc, db: db, enableAdmin: enableAdmin, @@ -148,7 +153,7 @@ func NewAPI( // Register the API's endpoints in the given router. func (api *API) Register(r *route.Router) { - instr := func(name string, f apiFunc) http.HandlerFunc { + wrap := func(f apiFunc) http.HandlerFunc { hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { setCORS(w) if data, err := f(r); err != nil { @@ -159,33 +164,34 @@ func (api *API) Register(r *route.Router) { w.WriteHeader(http.StatusNoContent) } }) - return api.ready(prometheus.InstrumentHandler(name, httputil.CompressionHandler{ + return api.ready(httputil.CompressionHandler{ Handler: hf, - })) + }.ServeHTTP) } - r.Options("/*path", instr("options", api.options)) + r.Options("/*path", wrap(api.options)) - r.Get("/query", instr("query", api.query)) - r.Post("/query", instr("query", api.query)) - r.Get("/query_range", instr("query_range", api.queryRange)) - r.Post("/query_range", instr("query_range", api.queryRange)) + r.Get("/query", wrap(api.query)) + r.Post("/query", wrap(api.query)) + r.Get("/query_range", wrap(api.queryRange)) + r.Post("/query_range", wrap(api.queryRange)) - r.Get("/label/:name/values", instr("label_values", api.labelValues)) + r.Get("/label/:name/values", wrap(api.labelValues)) - r.Get("/series", instr("series", api.series)) - r.Del("/series", instr("drop_series", api.dropSeries)) + r.Get("/series", wrap(api.series)) + r.Del("/series", wrap(api.dropSeries)) - r.Get("/targets", instr("targets", api.targets)) - r.Get("/alertmanagers", instr("alertmanagers", api.alertmanagers)) + r.Get("/targets", wrap(api.targets)) + r.Get("/alertmanagers", wrap(api.alertmanagers)) - r.Get("/status/config", instr("config", api.serveConfig)) - r.Post("/read", api.ready(prometheus.InstrumentHandler("read", http.HandlerFunc(api.remoteRead)))) + r.Get("/status/config", wrap(api.serveConfig)) + r.Get("/status/flags", wrap(api.serveFlags)) + r.Post("/read", api.ready(http.HandlerFunc(api.remoteRead))) // Admin APIs - r.Post("/admin/tsdb/delete_series", instr("delete_series", api.deleteSeries)) - r.Post("/admin/tsdb/clean_tombstones", instr("clean_tombstones", api.cleanTombstones)) - r.Post("/admin/tsdb/snapshot", instr("snapshot", api.snapshot)) + r.Post("/admin/tsdb/delete_series", wrap(api.deleteSeries)) + r.Post("/admin/tsdb/clean_tombstones", wrap(api.cleanTombstones)) + r.Post("/admin/tsdb/snapshot", wrap(api.snapshot)) } type queryData struct { @@ -222,7 +228,7 @@ func (api *API) query(r *http.Request) (interface{}, *apiError) { defer cancel() } - qry, err := api.QueryEngine.NewInstantQuery(r.FormValue("query"), ts) + qry, err := api.QueryEngine.NewInstantQuery(api.Queryable, r.FormValue("query"), ts) if err != nil { return nil, &apiError{errorBadData, err} } @@ -296,7 +302,7 @@ func (api *API) queryRange(r *http.Request) (interface{}, *apiError) { defer cancel() } - qry, err := api.QueryEngine.NewRangeQuery(r.FormValue("query"), start, end, step) + qry, err := api.QueryEngine.NewRangeQuery(api.Queryable, r.FormValue("query"), start, end, step) if err != nil { return nil, &apiError{errorBadData, err} } @@ -396,7 +402,7 @@ func (api *API) series(r *http.Request) (interface{}, *apiError) { var sets []storage.SeriesSet for _, mset := range matcherSets { - s, err := q.Select(mset...) + s, err := q.Select(nil, mset...) if err != nil { return nil, &apiError{errorExec, err} } @@ -433,14 +439,22 @@ type Target struct { Health scrape.TargetHealth `json:"health"` } +// DroppedTarget has the information for one target that was dropped during relabelling. +type DroppedTarget struct { + // Labels before any processing. + DiscoveredLabels map[string]string `json:"discoveredLabels"` +} + // TargetDiscovery has all the active targets. type TargetDiscovery struct { - ActiveTargets []*Target `json:"activeTargets"` + ActiveTargets []*Target `json:"activeTargets"` + DroppedTargets []*DroppedTarget `json:"droppedTargets"` } func (api *API) targets(r *http.Request) (interface{}, *apiError) { targets := api.targetRetriever.Targets() - res := &TargetDiscovery{ActiveTargets: make([]*Target, len(targets))} + droppedTargets := api.targetRetriever.DroppedTargets() + res := &TargetDiscovery{ActiveTargets: make([]*Target, len(targets)), DroppedTargets: make([]*DroppedTarget, len(droppedTargets))} for i, t := range targets { lastErrStr := "" @@ -459,12 +473,19 @@ func (api *API) targets(r *http.Request) (interface{}, *apiError) { } } + for i, t := range droppedTargets { + res.DroppedTargets[i] = &DroppedTarget{ + DiscoveredLabels: t.DiscoveredLabels().Map(), + } + } + return res, nil } // AlertmanagerDiscovery has all the active Alertmanagers. type AlertmanagerDiscovery struct { - ActiveAlertmanagers []*AlertmanagerTarget `json:"activeAlertmanagers"` + ActiveAlertmanagers []*AlertmanagerTarget `json:"activeAlertmanagers"` + DroppedAlertmanagers []*AlertmanagerTarget `json:"droppedAlertmanagers"` } // AlertmanagerTarget has info on one AM. @@ -474,12 +495,14 @@ type AlertmanagerTarget struct { func (api *API) alertmanagers(r *http.Request) (interface{}, *apiError) { urls := api.alertmanagerRetriever.Alertmanagers() - ams := &AlertmanagerDiscovery{ActiveAlertmanagers: make([]*AlertmanagerTarget, len(urls))} - + droppedURLS := api.alertmanagerRetriever.DroppedAlertmanagers() + ams := &AlertmanagerDiscovery{ActiveAlertmanagers: make([]*AlertmanagerTarget, len(urls)), DroppedAlertmanagers: make([]*AlertmanagerTarget, len(droppedURLS))} for i, url := range urls { ams.ActiveAlertmanagers[i] = &AlertmanagerTarget{URL: url.String()} } - + for i, url := range droppedURLS { + ams.DroppedAlertmanagers[i] = &AlertmanagerTarget{URL: url.String()} + } return ams, nil } @@ -494,6 +517,10 @@ func (api *API) serveConfig(r *http.Request) (interface{}, *apiError) { return cfg, nil } +func (api *API) serveFlags(r *http.Request) (interface{}, *apiError) { + return api.flagsMap, nil +} + func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { req, err := remote.DecodeReadRequest(r) if err != nil { @@ -537,7 +564,7 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { } } - set, err := querier.Select(filteredMatchers...) + set, err := querier.Select(nil, filteredMatchers...) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -630,6 +657,8 @@ func (api *API) snapshot(r *http.Request) (interface{}, *apiError) { if !api.enableAdmin { return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")} } + skipHead, _ := strconv.ParseBool(r.FormValue("skip_head")) + db := api.db() if db == nil { return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")} @@ -645,7 +674,7 @@ func (api *API) snapshot(r *http.Request) (interface{}, *apiError) { if err := os.MkdirAll(dir, 0777); err != nil { return nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %s", err)} } - if err := db.Snapshot(dir); err != nil { + if err := db.Snapshot(dir, !skipHead); err != nil { return nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %s", err)} } @@ -726,6 +755,7 @@ func respond(w http.ResponseWriter, data interface{}) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) + json := jsoniter.ConfigCompatibleWithStandardLibrary b, err := json.Marshal(&response{ Status: statusSuccess, Data: data, @@ -754,6 +784,7 @@ func respondError(w http.ResponseWriter, apiErr *apiError, data interface{}) { } w.WriteHeader(code) + json := jsoniter.ConfigCompatibleWithStandardLibrary b, err := json.Marshal(&response{ Status: statusError, ErrorType: apiErr.typ, @@ -790,3 +821,41 @@ func parseDuration(s string) (time.Duration, error) { } return 0, fmt.Errorf("cannot parse %q to a valid duration", s) } + +func init() { + jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty) +} + +func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) { + p := *((*promql.Point)(ptr)) + stream.WriteArrayStart() + // Write out the timestamp as a float divided by 1000. + // This is ~3x faster than converting to a float. + t := p.T + if t < 0 { + stream.WriteRaw(`-`) + t = -t + } + stream.WriteInt64(t / 1000) + fraction := t % 1000 + if fraction != 0 { + stream.WriteRaw(`.`) + if fraction < 100 { + stream.WriteRaw(`0`) + } + if fraction < 10 { + stream.WriteRaw(`0`) + } + stream.WriteInt64(fraction) + } + stream.WriteMore() + stream.WriteRaw(`"`) + stream.WriteFloat64(p.V) + stream.WriteRaw(`"`) + stream.WriteArrayEnd() + +} + +func marshalPointJSONIsEmpty(ptr unsafe.Pointer) bool { + return false +} diff --git a/vendor/gopkg.in/fsnotify.v1/.editorconfig b/vendor/gopkg.in/fsnotify/fsnotify.v1/.editorconfig similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/.editorconfig rename to vendor/gopkg.in/fsnotify/fsnotify.v1/.editorconfig diff --git a/vendor/gopkg.in/fsnotify.v1/.gitignore b/vendor/gopkg.in/fsnotify/fsnotify.v1/.gitignore similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/.gitignore rename to vendor/gopkg.in/fsnotify/fsnotify.v1/.gitignore diff --git a/vendor/gopkg.in/fsnotify.v1/.travis.yml b/vendor/gopkg.in/fsnotify/fsnotify.v1/.travis.yml similarity index 90% rename from vendor/gopkg.in/fsnotify.v1/.travis.yml rename to vendor/gopkg.in/fsnotify/fsnotify.v1/.travis.yml index 3a5c933bc0..981d1bb813 100644 --- a/vendor/gopkg.in/fsnotify.v1/.travis.yml +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/.travis.yml @@ -2,12 +2,14 @@ sudo: false language: go go: - - 1.6.3 + - 1.8.x + - 1.9.x - tip matrix: allow_failures: - go: tip + fast_finish: true before_script: - go get -u github.com/golang/lint/golint diff --git a/vendor/gopkg.in/fsnotify.v1/AUTHORS b/vendor/gopkg.in/fsnotify/fsnotify.v1/AUTHORS similarity index 88% rename from vendor/gopkg.in/fsnotify.v1/AUTHORS rename to vendor/gopkg.in/fsnotify/fsnotify.v1/AUTHORS index 0a5bf8f617..5ab5d41c54 100644 --- a/vendor/gopkg.in/fsnotify.v1/AUTHORS +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/AUTHORS @@ -8,8 +8,10 @@ # Please keep the list sorted. +Aaron L Adrien Bustany Amit Krishnan +Anmol Sethi Bjørn Erik Pedersen Bruno Bigras Caleb Spare @@ -26,6 +28,7 @@ Kelvin Fo Ken-ichirou MATSUZAWA Matt Layher Nathan Youngman +Nickolai Zeldovich Patrick Paul Hammond Pawel Knap @@ -33,12 +36,15 @@ Pieter Droogendijk Pursuit92 Riku Voipio Rob Figueiredo +Rodrigo Chiossi Slawek Ligus Soge Zhang Tiffany Jernigan Tilak Sharma +Tom Payne Travis Cline Tudor Golubenco +Vahe Khachikyan Yukang bronze1man debrando diff --git a/vendor/gopkg.in/fsnotify.v1/CHANGELOG.md b/vendor/gopkg.in/fsnotify/fsnotify.v1/CHANGELOG.md similarity index 93% rename from vendor/gopkg.in/fsnotify.v1/CHANGELOG.md rename to vendor/gopkg.in/fsnotify/fsnotify.v1/CHANGELOG.md index 40d7660d58..be4d7ea2c1 100644 --- a/vendor/gopkg.in/fsnotify.v1/CHANGELOG.md +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## v1.4.7 / 2018-01-09 + +* BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) +* Tests: Fix missing verb on format string (thanks @rchiossi) +* Linux: Fix deadlock in Remove (thanks @aarondl) +* Linux: Watch.Add improvements (avoid race, fix consistency, reduce garbage) (thanks @twpayne) +* Docs: Moved FAQ into the README (thanks @vahe) +* Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich) +* Docs: replace references to OS X with macOS + ## v1.4.2 / 2016-10-10 * Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) @@ -79,7 +89,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn ## v1.0.2 / 2014-08-17 -* [Fix] Missing create events on OS X. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) +* [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) * [Fix] Make ./path and path equivalent. (thanks @zhsso) ## v1.0.0 / 2014-08-15 @@ -142,7 +152,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn ## v0.9.2 / 2014-08-17 -* [Backport] Fix missing create events on OS X. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) +* [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) ## v0.9.1 / 2014-06-12 @@ -161,7 +171,7 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn ## v0.8.11 / 2013-11-02 * [Doc] Add Changelog [#72][] (thanks @nathany) -* [Doc] Spotlight and double modify events on OS X [#62][] (reported by @paulhammond) +* [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond) ## v0.8.10 / 2013-10-19 diff --git a/vendor/gopkg.in/fsnotify.v1/CONTRIBUTING.md b/vendor/gopkg.in/fsnotify/fsnotify.v1/CONTRIBUTING.md similarity index 93% rename from vendor/gopkg.in/fsnotify.v1/CONTRIBUTING.md rename to vendor/gopkg.in/fsnotify/fsnotify.v1/CONTRIBUTING.md index 6a81ba4890..828a60b24b 100644 --- a/vendor/gopkg.in/fsnotify.v1/CONTRIBUTING.md +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/CONTRIBUTING.md @@ -17,7 +17,7 @@ Please indicate that you have signed the CLA in your pull request. ### How fsnotify is Developed * Development is done on feature branches. -* Tests are run on BSD, Linux, OS X and Windows. +* Tests are run on BSD, Linux, macOS and Windows. * Pull requests are reviewed and [applied to master][am] using [hub][]. * Maintainers may modify or squash commits rather than asking contributors to. * To issue a new release, the maintainers will: @@ -44,7 +44,7 @@ This workflow is [thoroughly explained by Katrina Owen](https://splice.com/blog/ ### Testing -fsnotify uses build tags to compile different code on Linux, BSD, OS X, and Windows. +fsnotify uses build tags to compile different code on Linux, BSD, macOS, and Windows. Before doing a pull request, please do your best to test your changes on multiple platforms, and list which platforms you were able/unable to test on. @@ -58,7 +58,7 @@ To aid in cross-platform testing there is a Vagrantfile for Linux and BSD. Notice: fsnotify file system events won't trigger in shared folders. The tests get around this limitation by using the /tmp directory. -Right now there is no equivalent solution for Windows and OS X, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). +Right now there is no equivalent solution for Windows and macOS, but there are Windows VMs [freely available from Microsoft](http://www.modern.ie/en-us/virtualization-tools#downloads). ### Maintainers diff --git a/vendor/gopkg.in/fsnotify.v1/LICENSE b/vendor/gopkg.in/fsnotify/fsnotify.v1/LICENSE similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/LICENSE rename to vendor/gopkg.in/fsnotify/fsnotify.v1/LICENSE diff --git a/vendor/gopkg.in/fsnotify.v1/README.md b/vendor/gopkg.in/fsnotify/fsnotify.v1/README.md similarity index 59% rename from vendor/gopkg.in/fsnotify.v1/README.md rename to vendor/gopkg.in/fsnotify/fsnotify.v1/README.md index 3c891e349b..3993207413 100644 --- a/vendor/gopkg.in/fsnotify.v1/README.md +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/README.md @@ -8,14 +8,14 @@ fsnotify utilizes [golang.org/x/sys](https://godoc.org/golang.org/x/sys) rather go get -u golang.org/x/sys/... ``` -Cross platform: Windows, Linux, BSD and OS X. +Cross platform: Windows, Linux, BSD and macOS. |Adapter |OS |Status | |----------|----------|----------| |inotify |Linux 2.6.27 or later, Android\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| -|kqueue |BSD, OS X, iOS\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| +|kqueue |BSD, macOS, iOS\*|Supported [![Build Status](https://travis-ci.org/fsnotify/fsnotify.svg?branch=master)](https://travis-ci.org/fsnotify/fsnotify)| |ReadDirectoryChangesW|Windows|Supported [![Build status](https://ci.appveyor.com/api/projects/status/ivwjubaih4r0udeh/branch/master?svg=true)](https://ci.appveyor.com/project/NathanYoungman/fsnotify/branch/master)| -|FSEvents |OS X |[Planned](https://github.com/fsnotify/fsnotify/issues/11)| +|FSEvents |macOS |[Planned](https://github.com/fsnotify/fsnotify/issues/11)| |FEN |Solaris 11 |[In Progress](https://github.com/fsnotify/fsnotify/issues/12)| |fanotify |Linux 2.6.37+ | | |USN Journals |Windows |[Maybe](https://github.com/fsnotify/fsnotify/issues/53)| @@ -23,7 +23,7 @@ Cross platform: Windows, Linux, BSD and OS X. \* Android and iOS are untested. -Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) for usage. Consult the [Wiki](https://github.com/fsnotify/fsnotify/wiki) for the FAQ and further information. +Please see [the documentation](https://godoc.org/github.com/fsnotify/fsnotify) and consult the [FAQ](#faq) for usage information. ## API stability @@ -41,6 +41,35 @@ Please refer to [CONTRIBUTING][] before opening an issue or pull request. See [example_test.go](https://github.com/fsnotify/fsnotify/blob/master/example_test.go). +## FAQ + +**When a file is moved to another directory is it still being watched?** + +No (it shouldn't be, unless you are watching where it was moved to). + +**When I watch a directory, are all subdirectories watched as well?** + +No, you must add watches for any directory you want to watch (a recursive watcher is on the roadmap [#18][]). + +**Do I have to watch the Error and Event channels in a separate goroutine?** + +As of now, yes. Looking into making this single-thread friendly (see [howeyc #7][#7]) + +**Why am I receiving multiple events for the same file on OS X?** + +Spotlight indexing on OS X can result in multiple events (see [howeyc #62][#62]). A temporary workaround is to add your folder(s) to the *Spotlight Privacy settings* until we have a native FSEvents implementation (see [#11][]). + +**How many files can be watched at once?** + +There are OS-specific limits as to how many watches can be created: +* Linux: /proc/sys/fs/inotify/max_user_watches contains the limit, reaching this limit results in a "no space left on device" error. +* BSD / OSX: sysctl variables "kern.maxfiles" and "kern.maxfilesperproc", reaching these limits results in a "too many open files" error. + +[#62]: https://github.com/howeyc/fsnotify/issues/62 +[#18]: https://github.com/fsnotify/fsnotify/issues/18 +[#11]: https://github.com/fsnotify/fsnotify/issues/11 +[#7]: https://github.com/howeyc/fsnotify/issues/7 + [contributing]: https://github.com/fsnotify/fsnotify/blob/master/CONTRIBUTING.md ## Related Projects diff --git a/vendor/gopkg.in/fsnotify.v1/fen.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/fen.go similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/fen.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/fen.go diff --git a/vendor/gopkg.in/fsnotify.v1/fsnotify.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/fsnotify.go similarity index 91% rename from vendor/gopkg.in/fsnotify.v1/fsnotify.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/fsnotify.go index e7f55fee7a..190bf0de57 100644 --- a/vendor/gopkg.in/fsnotify.v1/fsnotify.go +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/fsnotify.go @@ -9,6 +9,7 @@ package fsnotify import ( "bytes" + "errors" "fmt" ) @@ -60,3 +61,6 @@ func (op Op) String() string { func (e Event) String() string { return fmt.Sprintf("%q: %s", e.Name, e.Op.String()) } + +// Common errors that can be reported by a watcher +var ErrEventOverflow = errors.New("fsnotify queue overflow") diff --git a/vendor/gopkg.in/fsnotify.v1/inotify.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/inotify.go similarity index 87% rename from vendor/gopkg.in/fsnotify.v1/inotify.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/inotify.go index f3b74c51f0..d9fd1b88a0 100644 --- a/vendor/gopkg.in/fsnotify.v1/inotify.go +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/inotify.go @@ -24,7 +24,6 @@ type Watcher struct { Events chan Event Errors chan error mu sync.Mutex // Map access - cv *sync.Cond // sync removing on rm_watch with IN_IGNORE fd int poller *fdPoller watches map[string]*watch // Map of inotify watches (key: path) @@ -56,7 +55,6 @@ func NewWatcher() (*Watcher, error) { done: make(chan struct{}), doneResp: make(chan struct{}), } - w.cv = sync.NewCond(&w.mu) go w.readEvents() return w, nil @@ -103,21 +101,23 @@ func (w *Watcher) Add(name string) error { var flags uint32 = agnosticEvents w.mu.Lock() - watchEntry, found := w.watches[name] - w.mu.Unlock() - if found { - watchEntry.flags |= flags - flags |= unix.IN_MASK_ADD + defer w.mu.Unlock() + watchEntry := w.watches[name] + if watchEntry != nil { + flags |= watchEntry.flags | unix.IN_MASK_ADD } wd, errno := unix.InotifyAddWatch(w.fd, name, flags) if wd == -1 { return errno } - w.mu.Lock() - w.watches[name] = &watch{wd: uint32(wd), flags: flags} - w.paths[wd] = name - w.mu.Unlock() + if watchEntry == nil { + w.watches[name] = &watch{wd: uint32(wd), flags: flags} + w.paths[wd] = name + } else { + watchEntry.wd = uint32(wd) + watchEntry.flags = flags + } return nil } @@ -135,6 +135,13 @@ func (w *Watcher) Remove(name string) error { if !ok { return fmt.Errorf("can't remove non-existent inotify watch for: %s", name) } + + // We successfully removed the watch if InotifyRmWatch doesn't return an + // error, we need to clean up our internal state to ensure it matches + // inotify's kernel state. + delete(w.paths, int(watch.wd)) + delete(w.watches, name) + // inotify_rm_watch will return EINVAL if the file has been deleted; // the inotify will already have been removed. // watches and pathes are deleted in ignoreLinux() implicitly and asynchronously @@ -152,13 +159,6 @@ func (w *Watcher) Remove(name string) error { return errno } - // wait until ignoreLinux() deleting maps - exists := true - for exists { - w.cv.Wait() - _, exists = w.watches[name] - } - return nil } @@ -245,13 +245,31 @@ func (w *Watcher) readEvents() { mask := uint32(raw.Mask) nameLen := uint32(raw.Len) + + if mask&unix.IN_Q_OVERFLOW != 0 { + select { + case w.Errors <- ErrEventOverflow: + case <-w.done: + return + } + } + // If the event happened to the watched directory or the watched file, the kernel // doesn't append the filename to the event, but we would like to always fill the // the "Name" field with a valid filename. We retrieve the path of the watch from // the "paths" map. w.mu.Lock() - name := w.paths[int(raw.Wd)] + name, ok := w.paths[int(raw.Wd)] + // IN_DELETE_SELF occurs when the file/directory being watched is removed. + // This is a sign to clean up the maps, otherwise we are no longer in sync + // with the inotify kernel state which has already deleted the watch + // automatically. + if ok && mask&unix.IN_DELETE_SELF == unix.IN_DELETE_SELF { + delete(w.paths, int(raw.Wd)) + delete(w.watches, name) + } w.mu.Unlock() + if nameLen > 0 { // Point "bytes" at the first byte of the filename bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) @@ -262,7 +280,7 @@ func (w *Watcher) readEvents() { event := newEvent(name, mask) // Send the events that are not ignored on the events channel - if !event.ignoreLinux(w, raw.Wd, mask) { + if !event.ignoreLinux(mask) { select { case w.Events <- event: case <-w.done: @@ -279,15 +297,9 @@ func (w *Watcher) readEvents() { // Certain types of events can be "ignored" and not sent over the Events // channel. Such as events marked ignore by the kernel, or MODIFY events // against files that do not exist. -func (e *Event) ignoreLinux(w *Watcher, wd int32, mask uint32) bool { +func (e *Event) ignoreLinux(mask uint32) bool { // Ignore anything the inotify API says to ignore if mask&unix.IN_IGNORED == unix.IN_IGNORED { - w.mu.Lock() - defer w.mu.Unlock() - name := w.paths[int(wd)] - delete(w.paths, int(wd)) - delete(w.watches, name) - w.cv.Broadcast() return true } diff --git a/vendor/gopkg.in/fsnotify.v1/inotify_poller.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/inotify_poller.go similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/inotify_poller.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/inotify_poller.go diff --git a/vendor/gopkg.in/fsnotify.v1/kqueue.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/kqueue.go similarity index 94% rename from vendor/gopkg.in/fsnotify.v1/kqueue.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/kqueue.go index c2b4acb18d..86e76a3d67 100644 --- a/vendor/gopkg.in/fsnotify.v1/kqueue.go +++ b/vendor/gopkg.in/fsnotify/fsnotify.v1/kqueue.go @@ -22,7 +22,7 @@ import ( type Watcher struct { Events chan Event Errors chan error - done chan bool // Channel for sending a "quit message" to the reader goroutine + done chan struct{} // Channel for sending a "quit message" to the reader goroutine kq int // File descriptor (as returned by the kqueue() syscall). @@ -56,7 +56,7 @@ func NewWatcher() (*Watcher, error) { externalWatches: make(map[string]bool), Events: make(chan Event), Errors: make(chan error), - done: make(chan bool), + done: make(chan struct{}), } go w.readEvents() @@ -71,10 +71,8 @@ func (w *Watcher) Close() error { return nil } w.isClosed = true - w.mu.Unlock() // copy paths to remove while locked - w.mu.Lock() var pathsToRemove = make([]string, 0, len(w.watches)) for name := range w.watches { pathsToRemove = append(pathsToRemove, name) @@ -82,15 +80,12 @@ func (w *Watcher) Close() error { w.mu.Unlock() // unlock before calling Remove, which also locks - var err error for _, name := range pathsToRemove { - if e := w.Remove(name); e != nil && err == nil { - err = e - } + w.Remove(name) } - // Send "quit" message to the reader goroutine: - w.done <- true + // send a "quit" message to the reader goroutine + close(w.done) return nil } @@ -266,17 +261,12 @@ func (w *Watcher) addWatch(name string, flags uint32) (string, error) { func (w *Watcher) readEvents() { eventBuffer := make([]unix.Kevent_t, 10) +loop: for { // See if there is a message on the "done" channel select { case <-w.done: - err := unix.Close(w.kq) - if err != nil { - w.Errors <- err - } - close(w.Events) - close(w.Errors) - return + break loop default: } @@ -284,7 +274,11 @@ func (w *Watcher) readEvents() { kevents, err := read(w.kq, eventBuffer, &keventWaitTime) // EINTR is okay, the syscall was interrupted before timeout expired. if err != nil && err != unix.EINTR { - w.Errors <- err + select { + case w.Errors <- err: + case <-w.done: + break loop + } continue } @@ -319,8 +313,12 @@ func (w *Watcher) readEvents() { if path.isDir && event.Op&Write == Write && !(event.Op&Remove == Remove) { w.sendDirectoryChangeEvents(event.Name) } else { - // Send the event on the Events channel - w.Events <- event + // Send the event on the Events channel. + select { + case w.Events <- event: + case <-w.done: + break loop + } } if event.Op&Remove == Remove { @@ -352,6 +350,18 @@ func (w *Watcher) readEvents() { kevents = kevents[1:] } } + + // cleanup + err := unix.Close(w.kq) + if err != nil { + // only way the previous loop breaks is if w.done was closed so we need to async send to w.Errors. + select { + case w.Errors <- err: + default: + } + } + close(w.Events) + close(w.Errors) } // newEvent returns an platform-independent Event based on kqueue Fflags. @@ -407,7 +417,11 @@ func (w *Watcher) sendDirectoryChangeEvents(dirPath string) { // Get all files files, err := ioutil.ReadDir(dirPath) if err != nil { - w.Errors <- err + select { + case w.Errors <- err: + case <-w.done: + return + } } // Search for new files @@ -428,7 +442,11 @@ func (w *Watcher) sendFileCreatedEventIfNew(filePath string, fileInfo os.FileInf w.mu.Unlock() if !doesExist { // Send create event - w.Events <- newCreateEvent(filePath) + select { + case w.Events <- newCreateEvent(filePath): + case <-w.done: + return + } } // like watchDirectoryFiles (but without doing another ReadDir) diff --git a/vendor/gopkg.in/fsnotify.v1/open_mode_bsd.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/open_mode_bsd.go similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/open_mode_bsd.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/open_mode_bsd.go diff --git a/vendor/gopkg.in/fsnotify.v1/open_mode_darwin.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/open_mode_darwin.go similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/open_mode_darwin.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/open_mode_darwin.go diff --git a/vendor/gopkg.in/fsnotify.v1/windows.go b/vendor/gopkg.in/fsnotify/fsnotify.v1/windows.go similarity index 100% rename from vendor/gopkg.in/fsnotify.v1/windows.go rename to vendor/gopkg.in/fsnotify/fsnotify.v1/windows.go From 89124d0621fa66726f2d776cb9e42f465ffb8eca Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 23 Mar 2018 18:01:42 +0000 Subject: [PATCH 2/9] Upgrade to Go 1.10 --- build-image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 1fa368d20c..5e28d7474c 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9.3-stretch +FROM golang:1.10.0-stretch RUN apt-get update && apt-get install -y python-requests python-yaml file jq unzip protobuf-compiler libprotobuf-dev && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN go clean -i net && \ From 03eb954a52808d37cb4f84fd87e5a24aafd69dae Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Fri, 23 Mar 2018 18:10:47 +0000 Subject: [PATCH 3/9] Fix up for API changes --- pkg/querier/querier.go | 10 +++++++--- pkg/querier/querier_test.go | 2 +- pkg/ruler/ruler.go | 8 ++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index cb5859d13a..414d4b98d1 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -3,12 +3,15 @@ package querier import ( "context" "net/http" + "time" "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/storage" + "github.com/weaveworks/cortex/pkg/ingester/client" "github.com/weaveworks/cortex/pkg/prom1/storage/metric" @@ -21,9 +24,10 @@ type ChunkStore interface { } // NewEngine creates a new promql.Engine for cortex. -func NewEngine(distributor Querier, chunkStore ChunkStore) *promql.Engine { +func NewEngine(distributor Querier, chunkStore ChunkStore, reg prometheus.Registerer, maxConcurrent int, timeout time.Duration) (*promql.Engine, storage.Queryable) { queryable := NewQueryable(distributor, chunkStore, false) - return promql.NewEngine(queryable, nil) + engine := promql.NewEngine(util.Logger, reg, maxConcurrent, timeout) + return engine, queryable } // NewQueryable creates a new Queryable for cortex. @@ -195,7 +199,7 @@ type mergeQuerier struct { metadataOnly bool } -func (mq mergeQuerier) Select(matchers ...*labels.Matcher) (storage.SeriesSet, error) { +func (mq mergeQuerier) Select(_ *storage.SelectParams, matchers ...*labels.Matcher) (storage.SeriesSet, error) { // TODO: Update underlying selectors to return errors directly. if mq.metadataOnly { return mq.selectMetadata(matchers...), nil diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 4e0c86c872..b5a465d7d1 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -110,7 +110,7 @@ func TestMergeQuerierSortsMetricLabels(t *testing.T) { } m, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "testmetric") require.NoError(t, err) - ss, err := mq.Select(m) + ss, err := mq.Select(nil, m) require.NoError(t, err) require.NoError(t, ss.Err()) ss.Next() diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 9a213bf771..2ac1b201af 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -26,6 +26,7 @@ import ( "github.com/prometheus/prometheus/notifier" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/rules" + "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/strutil" "golang.org/x/net/context" "golang.org/x/net/context/ctxhttp" @@ -123,6 +124,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { // Ruler evaluates rules. type Ruler struct { engine *promql.Engine + queryable storage.Queryable pusher Pusher alertURL *url.URL notifierCfg *config.Config @@ -201,8 +203,10 @@ func NewRuler(cfg Config, d *distributor.Distributor, c *chunk.Store) (*Ruler, e if err != nil { return nil, err } + engine, queryable := querier.NewEngine(d, c, prometheus.DefaultRegisterer, cfg.NumWorkers, cfg.GroupTimeout) return &Ruler{ - engine: querier.NewEngine(d, c), + engine: engine, + queryable: queryable, pusher: d, alertURL: cfg.ExternalURL.URL, notifierCfg: ncfg, @@ -283,7 +287,7 @@ func (r *Ruler) newGroup(ctx context.Context, userID string, item *workItem) (*r } opts := &rules.ManagerOptions{ Appendable: appendable, - QueryFunc: rules.EngineQueryFunc(r.engine), + QueryFunc: rules.EngineQueryFunc(r.engine, r.queryable), Context: ctx, ExternalURL: r.alertURL, NotifyFunc: sendAlerts(notifier, r.alertURL.String()), From 9ff4c803b4876be55f102f0920509210910d8756 Mon Sep 17 00:00:00 2001 From: Aaron Kirkbride Date: Mon, 26 Mar 2018 14:06:34 +0100 Subject: [PATCH 4/9] Update prometheus tsdb --- Gopkg.lock | 4 +- Gopkg.toml | 4 + vendor/github.com/prometheus/tsdb/block.go | 6 +- .../prometheus/tsdb/chunkenc/xor.go | 2 - vendor/github.com/prometheus/tsdb/compact.go | 57 ++++++---- vendor/github.com/prometheus/tsdb/db.go | 44 +++++++- .../prometheus/tsdb/fileutil/mmap_386.go | 5 + .../prometheus/tsdb/fileutil/mmap_amd64.go | 5 + .../prometheus/tsdb/fileutil/mmap_windows.go | 8 +- vendor/github.com/prometheus/tsdb/head.go | 8 +- .../github.com/prometheus/tsdb/index/index.go | 73 +++++++----- .../prometheus/tsdb/index/postings.go | 43 ++++---- vendor/github.com/prometheus/tsdb/repair.go | 104 ++++++++++++++++++ 13 files changed, 273 insertions(+), 90 deletions(-) create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go create mode 100644 vendor/github.com/prometheus/tsdb/repair.go diff --git a/Gopkg.lock b/Gopkg.lock index 13dc0eea60..4c05f0f57f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -750,7 +750,7 @@ "index", "labels" ] - revision = "467948f3c3f2f6f4ed9881afba27dd3ae24393eb" + revision = "195bc0d286b077f5633eb4bda76743620588b2fa" [[projects]] branch = "master" @@ -1190,6 +1190,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "5ce05e0826d6ffafc0d52bdadf741b4db5dc53d5db655ce87785717e67813146" + inputs-digest = "a45496e76b9d180c0d63003459b5cd153c8bc0c430ea97e6f8e3902fb3c9ee8c" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 65c60bc01c..3145800d52 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -19,6 +19,10 @@ name = "github.com/prometheus/prometheus" branch = "master" +[[constraint]] + name = "github.com/prometheus/tsdb" + branch = "master" + [[override]] name = "k8s.io/client-go" revision = "3627aeb7d4f6ade38f995d2c923e459146493c7e" diff --git a/vendor/github.com/prometheus/tsdb/block.go b/vendor/github.com/prometheus/tsdb/block.go index 8cf28fe0eb..7dc3af9d80 100644 --- a/vendor/github.com/prometheus/tsdb/block.go +++ b/vendor/github.com/prometheus/tsdb/block.go @@ -195,7 +195,7 @@ func readMetaFile(dir string) (*BlockMeta, error) { if err := json.Unmarshal(b, &m); err != nil { return nil, err } - if m.Version != 1 && m.Version != 2 { + if m.Version != 1 { return nil, errors.Errorf("unexpected meta file version %d", m.Version) } @@ -203,6 +203,8 @@ func readMetaFile(dir string) (*BlockMeta, error) { } func writeMetaFile(dir string, meta *BlockMeta) error { + meta.Version = 1 + // Make any changes to the file appear atomic. path := filepath.Join(dir, metaFilename) tmp := path + ".tmp" @@ -253,7 +255,7 @@ func OpenBlock(dir string, pool chunkenc.Pool) (*Block, error) { if err != nil { return nil, err } - ir, err := index.NewFileReader(filepath.Join(dir, "index"), meta.Version) + ir, err := index.NewFileReader(filepath.Join(dir, "index")) if err != nil { return nil, err } diff --git a/vendor/github.com/prometheus/tsdb/chunkenc/xor.go b/vendor/github.com/prometheus/tsdb/chunkenc/xor.go index b3f5b5135b..ff690f2228 100644 --- a/vendor/github.com/prometheus/tsdb/chunkenc/xor.go +++ b/vendor/github.com/prometheus/tsdb/chunkenc/xor.go @@ -89,7 +89,6 @@ func (c *XORChunk) Appender() (Appender, error) { } a := &xorAppender{ - c: c, b: c.b, t: it.t, v: it.val, @@ -119,7 +118,6 @@ func (c *XORChunk) Iterator() Iterator { } type xorAppender struct { - c *XORChunk b *bstream t int64 diff --git a/vendor/github.com/prometheus/tsdb/compact.go b/vendor/github.com/prometheus/tsdb/compact.go index f96e643945..16a3bd7471 100644 --- a/vendor/github.com/prometheus/tsdb/compact.go +++ b/vendor/github.com/prometheus/tsdb/compact.go @@ -14,6 +14,7 @@ package tsdb import ( + "fmt" "io" "math/rand" "os" @@ -33,7 +34,7 @@ import ( "github.com/prometheus/tsdb/labels" ) -// ExponentialBlockRanges returns the time ranges based on the stepSize +// ExponentialBlockRanges returns the time ranges based on the stepSize. func ExponentialBlockRanges(minSize int64, steps, stepSize int) []int64 { ranges := make([]int64, 0, steps) curRange := minSize @@ -91,8 +92,8 @@ func newCompactorMetrics(r prometheus.Registerer) *compactorMetrics { Help: "Total number of compactions that failed for the partition.", }) m.duration = prometheus.NewHistogram(prometheus.HistogramOpts{ - Name: "prometheus_tsdb_compaction_duration", - Help: "Duration of compaction runs.", + Name: "prometheus_tsdb_compaction_duration_seconds", + Help: "Duration of compaction runs", Buckets: prometheus.ExponentialBuckets(1, 2, 10), }) m.chunkSize = prometheus.NewHistogram(prometheus.HistogramOpts{ @@ -151,9 +152,11 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) { if err != nil { return nil, err } + if len(dirs) < 1 { + return nil, nil + } var dms []dirMeta - for _, dir := range dirs { meta, err := readMetaFile(dir) if err != nil { @@ -169,6 +172,10 @@ func (c *LeveledCompactor) plan(dms []dirMeta) ([]string, error) { return dms[i].meta.MinTime < dms[j].meta.MinTime }) + // We do not include a recently created block with max(minTime), so the block which was just created from WAL. + // This gives users a window of a full block size to piece-wise backup new data without having to care about data overlap. + dms = dms[:len(dms)-1] + var res []string for _, dm := range c.selectDirs(dms) { res = append(res, dm.dir) @@ -209,7 +216,7 @@ func (c *LeveledCompactor) selectDirs(ds []dirMeta) []dirMeta { Outer: for _, p := range parts { - // Donot select the range if it has a block whose compaction failed. + // Do not select the range if it has a block whose compaction failed. for _, dm := range p { if dm.meta.Compaction.Failed { continue Outer @@ -306,9 +313,12 @@ func compactBlockMetas(uid ulid.ULID, blocks ...*BlockMeta) *BlockMeta { // Compact creates a new block in the compactor's directory from the blocks in the // provided directories. func (c *LeveledCompactor) Compact(dest string, dirs ...string) (uid ulid.ULID, err error) { - var blocks []BlockReader - var bs []*Block - var metas []*BlockMeta + var ( + blocks []BlockReader + bs []*Block + metas []*BlockMeta + uids []string + ) for _, d := range dirs { b, err := OpenBlock(d, c.chunkPool) @@ -325,13 +335,23 @@ func (c *LeveledCompactor) Compact(dest string, dirs ...string) (uid ulid.ULID, metas = append(metas, meta) blocks = append(blocks, b) bs = append(bs, b) + uids = append(uids, meta.ULID.String()) } entropy := rand.New(rand.NewSource(time.Now().UnixNano())) uid = ulid.MustNew(ulid.Now(), entropy) - err = c.write(dest, compactBlockMetas(uid, metas...), blocks...) + meta := compactBlockMetas(uid, metas...) + err = c.write(dest, meta, blocks...) if err == nil { + level.Info(c.logger).Log( + "msg", "compact blocks", + "count", len(blocks), + "mint", meta.MinTime, + "maxt", meta.MaxTime, + "ulid", meta.ULID, + "sources", fmt.Sprintf("%v", uids), + ) return uid, nil } @@ -359,7 +379,13 @@ func (c *LeveledCompactor) Write(dest string, b BlockReader, mint, maxt int64) ( meta.Compaction.Level = 1 meta.Compaction.Sources = []ulid.ULID{uid} - return uid, c.write(dest, meta, b) + err := c.write(dest, meta, b) + if err != nil { + return uid, err + } + + level.Info(c.logger).Log("msg", "write block", "mint", meta.MinTime, "maxt", meta.MaxTime, "ulid", meta.ULID) + return uid, nil } // instrumentedChunkWriter is used for level 1 compactions to record statistics @@ -384,8 +410,6 @@ func (w *instrumentedChunkWriter) WriteChunks(chunks ...chunks.Meta) error { // write creates a new block that is the union of the provided blocks into dir. // It cleans up all files of the old blocks after completing successfully. func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockReader) (err error) { - level.Info(c.logger).Log("msg", "compact blocks", "count", len(blocks), "mint", meta.MinTime, "maxt", meta.MaxTime) - dir := filepath.Join(dest, meta.ULID.String()) tmp := dir + ".tmp" @@ -428,7 +452,6 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockRe } indexw, err := index.NewWriter(filepath.Join(tmp, indexFilename)) - meta.Version = indexw.Version if err != nil { return errors.Wrap(err, "open index writer") } @@ -467,7 +490,7 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockRe return errors.Wrap(err, "sync temporary dir file") } - // close temp dir before rename block dir(for windows platform) + // Close temp dir before rename block dir (for windows platform). if err = df.Close(); err != nil { return errors.Wrap(err, "close temporary dir") } @@ -477,6 +500,7 @@ func (c *LeveledCompactor) write(dest string, meta *BlockMeta, blocks ...BlockRe if err := renameFile(tmp, dir); err != nil { return errors.Wrap(err, "rename block dir") } + return nil } @@ -713,11 +737,6 @@ type compactionMerger struct { intervals Intervals } -type compactionSeries struct { - labels labels.Labels - chunks []*chunks.Meta -} - func newCompactionMerger(a, b ChunkSeriesSet) (*compactionMerger, error) { c := &compactionMerger{ a: a, diff --git a/vendor/github.com/prometheus/tsdb/db.go b/vendor/github.com/prometheus/tsdb/db.go index b659165f17..489f3a12e5 100644 --- a/vendor/github.com/prometheus/tsdb/db.go +++ b/vendor/github.com/prometheus/tsdb/db.go @@ -42,7 +42,7 @@ import ( ) // DefaultOptions used for the DB. They are sane for setups using -// millisecond precision timestampdb. +// millisecond precision timestamps. var DefaultOptions = &Options{ WALFlushInterval: 5 * time.Second, RetentionDuration: 15 * 24 * 60 * 60 * 1000, // 15 days in milliseconds @@ -122,6 +122,8 @@ type dbMetrics struct { reloads prometheus.Counter reloadsFailed prometheus.Counter compactionsTriggered prometheus.Counter + cutoffs prometheus.Counter + cutoffsFailed prometheus.Counter tombCleanTimer prometheus.Histogram } @@ -148,6 +150,14 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { Name: "prometheus_tsdb_compactions_triggered_total", Help: "Total number of triggered compactions for the partition.", }) + m.cutoffs = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "prometheus_tsdb_retention_cutoffs_total", + Help: "Number of times the database cut off block data from disk.", + }) + m.cutoffsFailed = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "prometheus_tsdb_retention_cutoffs_failures_total", + Help: "Number of times the database failed to cut off block data from disk.", + }) m.tombCleanTimer = prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "prometheus_tsdb_tombstone_cleanup_seconds", Help: "The time taken to recompact blocks to remove tombstones.", @@ -158,6 +168,8 @@ func newDBMetrics(db *DB, r prometheus.Registerer) *dbMetrics { m.loadedBlocks, m.reloads, m.reloadsFailed, + m.cutoffs, + m.cutoffsFailed, m.compactionsTriggered, m.tombCleanTimer, ) @@ -176,6 +188,10 @@ func Open(dir string, l log.Logger, r prometheus.Registerer, opts *Options) (db if opts == nil { opts = DefaultOptions } + // Fixup bad format written by Prometheus 2.1. + if err := repairBadIndexVersion(l, dir); err != nil { + return nil, err + } db = &DB{ dir: dir, @@ -277,7 +293,17 @@ func (db *DB) run() { } } -func (db *DB) retentionCutoff() (bool, error) { +func (db *DB) retentionCutoff() (b bool, err error) { + defer func() { + if !b && err == nil { + // no data had to be cut off. + return + } + db.metrics.cutoffs.Inc() + if err != nil { + db.metrics.cutoffsFailed.Inc() + } + }() if db.opts.RetentionDuration == 0 { return false, nil } @@ -299,7 +325,11 @@ func (db *DB) retentionCutoff() (bool, error) { } // This will close the dirs and then delete the dirs. - return len(dirs) > 0, db.reload(dirs...) + if len(dirs) > 0 { + return true, db.reload(dirs...) + } + + return false, nil } // Appender opens a new appender against the database. @@ -603,8 +633,9 @@ func (db *DB) EnableCompactions() { level.Info(db.logger).Log("msg", "compactions enabled") } -// Snapshot writes the current data to the directory. -func (db *DB) Snapshot(dir string) error { +// Snapshot writes the current data to the directory. If withHead is set to true it +// will create a new block containing all data that's currently in the memory buffer/WAL. +func (db *DB) Snapshot(dir string, withHead bool) error { if dir == db.dir { return errors.Errorf("cannot snapshot into base directory") } @@ -625,6 +656,9 @@ func (db *DB) Snapshot(dir string) error { return errors.Wrapf(err, "error snapshotting block: %s", b.Dir()) } } + if !withHead { + return nil + } _, err := db.compactor.Write(dir, db.head, db.head.MinTime(), db.head.MaxTime()) return errors.Wrap(err, "snapshot head block") } diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go new file mode 100644 index 0000000000..156f81b638 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go @@ -0,0 +1,5 @@ +// +build windows + +package fileutil + +const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go new file mode 100644 index 0000000000..4025dbfcb9 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go @@ -0,0 +1,5 @@ +// +build windows + +package fileutil + +const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go index 3bee807c2c..b942264123 100644 --- a/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go @@ -19,14 +19,14 @@ import ( "unsafe" ) -func mmap(f *os.File, sz int) ([]byte, error) { - low, high := uint32(sz), uint32(sz>>32) +func mmap(f *os.File, size int) ([]byte, error) { + low, high := uint32(size), uint32(size>>32) h, errno := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil) if h == 0 { return nil, os.NewSyscallError("CreateFileMapping", errno) } - addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(sz)) + addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(size)) if addr == 0 { return nil, os.NewSyscallError("MapViewOfFile", errno) } @@ -35,7 +35,7 @@ func mmap(f *os.File, sz int) ([]byte, error) { return nil, os.NewSyscallError("CloseHandle", err) } - return (*[1 << 30]byte)(unsafe.Pointer(addr))[:sz], nil + return (*[maxMapSize]byte)(unsafe.Pointer(addr))[:size], nil } func munmap(b []byte) error { diff --git a/vendor/github.com/prometheus/tsdb/head.go b/vendor/github.com/prometheus/tsdb/head.go index 4e1bea9579..c76c139d5a 100644 --- a/vendor/github.com/prometheus/tsdb/head.go +++ b/vendor/github.com/prometheus/tsdb/head.go @@ -265,7 +265,7 @@ func (h *Head) ReadWAL() error { // TODO(fabxc): series entries spread between samples can starve the sample workers. // Even with bufferd channels, this can impact startup time with lots of series churn. - // We must not pralellize series creation itself but could make the indexing asynchronous. + // We must not paralellize series creation itself but could make the indexing asynchronous. seriesFunc := func(series []RefSeries) { for _, s := range series { h.getOrCreateWithID(s.Ref, s.Labels.Hash(), s.Labels) @@ -762,10 +762,6 @@ func (c *safeChunk) Iterator() chunkenc.Iterator { return it } -// func (c *safeChunk) Appender() (chunks.Appender, error) { panic("illegal") } -// func (c *safeChunk) Bytes() []byte { panic("illegal") } -// func (c *safeChunk) Encoding() chunks.Encoding { panic("illegal") } - type headIndexReader struct { head *Head mint, maxt int64 @@ -1259,7 +1255,7 @@ func (s *memSeries) iterator(id int) chunkenc.Iterator { if id-s.firstChunkID < len(s.chunks)-1 { return c.chunk.Iterator() } - // Serve the last 4 samples for the last chunk from the series buffer + // Serve the last 4 samples for the last chunk from the sample buffer // as their compressed bytes may be mutated by added samples. it := &memSafeIterator{ Iterator: c.chunk.Iterator(), diff --git a/vendor/github.com/prometheus/tsdb/index/index.go b/vendor/github.com/prometheus/tsdb/index/index.go index f6b680b365..3cad716c07 100644 --- a/vendor/github.com/prometheus/tsdb/index/index.go +++ b/vendor/github.com/prometheus/tsdb/index/index.go @@ -37,6 +37,7 @@ const ( MagicIndex = 0xBAAAD700 indexFormatV1 = 1 + indexFormatV2 = 2 ) type indexWriterSeries struct { @@ -135,7 +136,7 @@ type indexTOC struct { postingsTable uint64 } -// NewWriter returns a new Writer to the given filename. +// NewWriter returns a new Writer to the given filename. It serializes data in format version 2. func NewWriter(fn string) (*Writer, error) { dir := filepath.Dir(fn) @@ -143,7 +144,11 @@ func NewWriter(fn string) (*Writer, error) { if err != nil { return nil, err } - defer df.Close() // close for flatform windows + defer df.Close() // Close for platform windows. + + if err := os.RemoveAll(fn); err != nil { + return nil, errors.Wrap(err, "remove any existing index at path") + } f, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY, 0666) if err != nil { @@ -168,8 +173,6 @@ func NewWriter(fn string) (*Writer, error) { symbols: make(map[string]uint32, 1<<13), seriesOffsets: make(map[uint64]uint64, 1<<16), crc32: newCRC32(), - - Version: 2, } if err := iw.writeMeta(); err != nil { return nil, err @@ -195,7 +198,7 @@ func (w *Writer) write(bufs ...[]byte) error { return nil } -// addPadding adds zero byte padding until the file size is a multiple size_unit. +// addPadding adds zero byte padding until the file size is a multiple size. func (w *Writer) addPadding(size int) error { p := w.pos % uint64(size) if p == 0 { @@ -249,7 +252,7 @@ func (w *Writer) ensureStage(s indexWriterStage) error { func (w *Writer) writeMeta() error { w.buf1.reset() w.buf1.putBE32(MagicIndex) - w.buf1.putByte(indexFormatV1) + w.buf1.putByte(indexFormatV2) return w.write(w.buf1.get()) } @@ -266,7 +269,13 @@ func (w *Writer) AddSeries(ref uint64, lset labels.Labels, chunks ...chunks.Meta if _, ok := w.seriesOffsets[ref]; ok { return errors.Errorf("series with reference %d already added", ref) } + // We add padding to 16 bytes to increase the addressable space we get through 4 byte + // series references. w.addPadding(16) + + if w.pos%16 != 0 { + return errors.Errorf("series write not 16-byte aligned at %d", w.pos) + } w.seriesOffsets[ref] = w.pos / 16 w.buf2.reset() @@ -530,8 +539,8 @@ type Reader struct { c io.Closer // Cached hashmaps of section offsets. - labels map[string]uint32 - postings map[labels.Label]uint32 + labels map[string]uint64 + postings map[labels.Label]uint64 // Cache of read symbols. Strings that are returned when reading from the // block are always backed by true strings held in here rather than // strings that are backed by byte slices from the mmap'd index file. This @@ -572,42 +581,43 @@ func (b realByteSlice) Sub(start, end int) ByteSlice { return b[start:end] } -// NewReader returns a new IndexReader on the given byte slice. -func NewReader(b ByteSlice, version int) (*Reader, error) { - return newReader(b, nil, version) +// NewReader returns a new IndexReader on the given byte slice. It automatically +// handles different format versions. +func NewReader(b ByteSlice) (*Reader, error) { + return newReader(b, nil) } // NewFileReader returns a new index reader against the given index file. -func NewFileReader(path string, version int) (*Reader, error) { +func NewFileReader(path string) (*Reader, error) { f, err := fileutil.OpenMmapFile(path) if err != nil { return nil, err } - return newReader(realByteSlice(f.Bytes()), f, version) + return newReader(realByteSlice(f.Bytes()), f) } -func newReader(b ByteSlice, c io.Closer, version int) (*Reader, error) { - if version != 1 && version != 2 { - return nil, errors.Errorf("unexpected file version %d", version) - } - +func newReader(b ByteSlice, c io.Closer) (*Reader, error) { r := &Reader{ b: b, c: c, symbols: map[uint32]string{}, - labels: map[string]uint32{}, - postings: map[labels.Label]uint32{}, + labels: map[string]uint64{}, + postings: map[labels.Label]uint64{}, crc32: newCRC32(), - version: version, } - // Verify magic number. - if b.Len() < 4 { + // Verify header. + if b.Len() < 5 { return nil, errors.Wrap(errInvalidSize, "index header") } if m := binary.BigEndian.Uint32(r.b.Range(0, 4)); m != MagicIndex { return nil, errors.Errorf("invalid magic number %x", m) } + r.version = int(r.b.Range(4, 5)[0]) + + if r.version != 1 && r.version != 2 { + return nil, errors.Errorf("unknown index file version %d", r.version) + } if err := r.readTOC(); err != nil { return nil, errors.Wrap(err, "read TOC") @@ -617,7 +627,7 @@ func newReader(b ByteSlice, c io.Closer, version int) (*Reader, error) { } var err error - err = r.readOffsetTable(r.toc.labelIndicesTable, func(key []string, off uint32) error { + err = r.readOffsetTable(r.toc.labelIndicesTable, func(key []string, off uint64) error { if len(key) != 1 { return errors.Errorf("unexpected key length %d", len(key)) } @@ -627,7 +637,7 @@ func newReader(b ByteSlice, c io.Closer, version int) (*Reader, error) { if err != nil { return nil, errors.Wrap(err, "read label index table") } - err = r.readOffsetTable(r.toc.postingsTable, func(key []string, off uint32) error { + err = r.readOffsetTable(r.toc.postingsTable, func(key []string, off uint64) error { if len(key) != 2 { return errors.Errorf("unexpected key length %d", len(key)) } @@ -643,6 +653,11 @@ func newReader(b ByteSlice, c io.Closer, version int) (*Reader, error) { return r, nil } +// Version returns the file format version of the underlying index. +func (r *Reader) Version() int { + return r.version +} + // Range marks a byte range. type Range struct { Start, End int64 @@ -780,7 +795,7 @@ func (r *Reader) readSymbols(off int) error { // readOffsetTable reads an offset table at the given position calls f for each // found entry.f // If f returns an error it stops decoding and returns the received error, -func (r *Reader) readOffsetTable(off uint64, f func([]string, uint32) error) error { +func (r *Reader) readOffsetTable(off uint64, f func([]string, uint64) error) error { d := r.decbufAt(int(off)) cnt := d.be32() @@ -791,7 +806,7 @@ func (r *Reader) readOffsetTable(off uint64, f func([]string, uint32) error) err for i := 0; i < keyCount; i++ { keys = append(keys, d.uvarintStr()) } - o := uint32(d.uvarint()) + o := d.uvarint64() if d.err() != nil { break } @@ -880,8 +895,10 @@ func (r *Reader) LabelIndices() ([][]string, error) { // Series reads the series with the given ID and writes its labels and chunks into lbls and chks. func (r *Reader) Series(id uint64, lbls *labels.Labels, chks *[]chunks.Meta) error { offset := id + // In version 2 series IDs are no longer exact references but series are 16-byte padded + // and the ID is the multiple of 16 of the actual position. if r.version == 2 { - offset = 16 * id + offset = id * 16 } d := r.decbufUvarintAt(int(offset)) if d.err() != nil { diff --git a/vendor/github.com/prometheus/tsdb/index/postings.go b/vendor/github.com/prometheus/tsdb/index/postings.go index f942edaad7..53449f341a 100644 --- a/vendor/github.com/prometheus/tsdb/index/postings.go +++ b/vendor/github.com/prometheus/tsdb/index/postings.go @@ -445,32 +445,31 @@ func (rp *removedPostings) Next() bool { rp.rok = rp.remove.Next() rp.initialized = true } + for { + if !rp.fok { + return false + } - if !rp.fok { - return false - } - - if !rp.rok { - rp.cur = rp.full.At() - rp.fok = rp.full.Next() - return true - } + if !rp.rok { + rp.cur = rp.full.At() + rp.fok = rp.full.Next() + return true + } - fcur, rcur := rp.full.At(), rp.remove.At() - if fcur < rcur { - rp.cur = fcur - rp.fok = rp.full.Next() + fcur, rcur := rp.full.At(), rp.remove.At() + if fcur < rcur { + rp.cur = fcur + rp.fok = rp.full.Next() - return true - } else if rcur < fcur { - // Forward the remove postings to the right position. - rp.rok = rp.remove.Seek(fcur) - } else { - // Skip the current posting. - rp.fok = rp.full.Next() + return true + } else if rcur < fcur { + // Forward the remove postings to the right position. + rp.rok = rp.remove.Seek(fcur) + } else { + // Skip the current posting. + rp.fok = rp.full.Next() + } } - - return rp.Next() } func (rp *removedPostings) Seek(id uint64) bool { diff --git a/vendor/github.com/prometheus/tsdb/repair.go b/vendor/github.com/prometheus/tsdb/repair.go new file mode 100644 index 0000000000..2722609a71 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/repair.go @@ -0,0 +1,104 @@ +package tsdb + +import ( + "encoding/json" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/tsdb/fileutil" +) + +// repairBadIndexVersion repairs an issue in index and meta.json persistence introduced in +// commit 129773b41a565fde5156301e37f9a87158030443. +func repairBadIndexVersion(logger log.Logger, dir string) error { + // All blocks written by Prometheus 2.1 with a meta.json version of 2 are affected. + // We must actually set the index file version to 2 and revert the meta.json version back to 1. + subdirs, err := fileutil.ReadDir(dir) + if err != nil { + return err + } + for _, d := range subdirs { + // Skip non-block dirs. + if _, err := ulid.Parse(d); err != nil { + continue + } + d = path.Join(dir, d) + + meta, err := readBogusMetaFile(d) + if err != nil { + return err + } + if meta.Version == 1 { + level.Info(logger).Log( + "msg", "found healthy block", + "mint", meta.MinTime, + "maxt", meta.MaxTime, + "ulid", meta.ULID, + ) + continue + } + level.Info(logger).Log( + "msg", "fixing broken block", + "mint", meta.MinTime, + "maxt", meta.MaxTime, + "ulid", meta.ULID, + ) + + repl, err := os.Create(filepath.Join(d, "index.repaired")) + if err != nil { + return err + } + broken, err := os.Open(filepath.Join(d, "index")) + if err != nil { + return err + } + if _, err := io.Copy(repl, broken); err != nil { + return err + } + // Set the 5th byte to 2 to indiciate the correct file format version. + if _, err := repl.WriteAt([]byte{2}, 4); err != nil { + return err + } + if err := fileutil.Fsync(repl); err != nil { + return err + } + if err := repl.Close(); err != nil { + return err + } + if err := broken.Close(); err != nil { + return err + } + if err := renameFile(repl.Name(), broken.Name()); err != nil { + return err + } + // Reset version of meta.json to 1. + meta.Version = 1 + if err := writeMetaFile(d, meta); err != nil { + return err + } + } + return nil +} + +func readBogusMetaFile(dir string) (*BlockMeta, error) { + b, err := ioutil.ReadFile(filepath.Join(dir, metaFilename)) + if err != nil { + return nil, err + } + var m BlockMeta + + if err := json.Unmarshal(b, &m); err != nil { + return nil, err + } + if m.Version != 1 && m.Version != 2 { + return nil, errors.Errorf("unexpected meta file version %d", m.Version) + } + return &m, nil +} From a1c542a797e30899d4c7b5f2b891e4c405ff4c1e Mon Sep 17 00:00:00 2001 From: Aaron Kirkbride Date: Mon, 26 Mar 2018 14:34:26 +0100 Subject: [PATCH 5/9] Update promql.NewEngine and v1.NewAPI --- cmd/lite/main.go | 6 +++++- cmd/querier/main.go | 6 +++++- pkg/querier/dummy.go | 6 ++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/lite/main.go b/cmd/lite/main.go index 7b4f013bb0..a393e0d882 100644 --- a/cmd/lite/main.go +++ b/cmd/lite/main.go @@ -4,6 +4,7 @@ import ( "flag" "net/http" "os" + "time" "github.com/go-kit/kit/log/level" "github.com/opentracing-contrib/go-stdlib/nethttp" @@ -147,13 +148,16 @@ func main() { sampleQueryable := querier.NewQueryable(dist, chunkStore, false) metadataQueryable := querier.NewQueryable(dist, chunkStore, true) - engine := promql.NewEngine(sampleQueryable, nil) + maxConcurrent := 20 + timeout := 2 * time.Minute + engine := promql.NewEngine(util.Logger, nil, maxConcurrent, timeout) api := v1.NewAPI( engine, metadataQueryable, querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{}, func() config.Config { return config.Config{} }, + map[string]string{}, // TODO: include configuration flags func(f http.HandlerFunc) http.HandlerFunc { return f }, func() *tsdb.DB { return nil }, // Only needed for admin APIs. false, // Disable admin APIs. diff --git a/cmd/querier/main.go b/cmd/querier/main.go index 36b544e05f..f0f0890af1 100644 --- a/cmd/querier/main.go +++ b/cmd/querier/main.go @@ -4,6 +4,7 @@ import ( "flag" "net/http" "os" + "time" "google.golang.org/grpc" @@ -90,13 +91,16 @@ func main() { sampleQueryable := querier.NewQueryable(dist, chunkStore, false) metadataQueryable := querier.NewQueryable(dist, chunkStore, true) - engine := promql.NewEngine(sampleQueryable, nil) + maxConcurrent := 20 + timeout := 2 * time.Minute + engine := promql.NewEngine(util.Logger, nil, maxConcurrent, timeout) api := v1.NewAPI( engine, metadataQueryable, querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{}, func() config.Config { return config.Config{} }, + map[string]string{}, // TODO: include configuration flags func(f http.HandlerFunc) http.HandlerFunc { return f }, func() *tsdb.DB { return nil }, // Only needed for admin APIs. false, // Disable admin APIs. diff --git a/pkg/querier/dummy.go b/pkg/querier/dummy.go index fffde45d9b..a140eae692 100644 --- a/pkg/querier/dummy.go +++ b/pkg/querier/dummy.go @@ -12,8 +12,14 @@ type DummyTargetRetriever struct{} // Targets implements TargetRetriever. func (r DummyTargetRetriever) Targets() []*scrape.Target { return nil } +// DroppedTargets implements TargetRetriever. +func (r DummyTargetRetriever) DroppedTargets() []*scrape.Target { return nil } + // DummyAlertmanagerRetriever implements AlertmanagerRetriever. type DummyAlertmanagerRetriever struct{} // Alertmanagers implements AlertmanagerRetriever. func (r DummyAlertmanagerRetriever) Alertmanagers() []*url.URL { return nil } + +// DroppedAlertmanagers implements AlertmanagerRetriever. +func (r DummyAlertmanagerRetriever) DroppedAlertmanagers() []*url.URL { return nil } From a0bac5970cab25a3005fd8e012adb4af98b7f55d Mon Sep 17 00:00:00 2001 From: Aaron Kirkbride Date: Mon, 26 Mar 2018 15:26:55 +0100 Subject: [PATCH 6/9] Create querier.Config --- cmd/lite/main.go | 12 +++++------- cmd/querier/main.go | 8 +++----- cmd/ruler/main.go | 7 +++++-- pkg/querier/querier.go | 13 +++++++++++++ pkg/ruler/ruler.go | 5 +---- pkg/ruler/ruler_test.go | 5 ++++- 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/cmd/lite/main.go b/cmd/lite/main.go index a393e0d882..9269fe7729 100644 --- a/cmd/lite/main.go +++ b/cmd/lite/main.go @@ -4,7 +4,6 @@ import ( "flag" "net/http" "os" - "time" "github.com/go-kit/kit/log/level" "github.com/opentracing-contrib/go-stdlib/nethttp" @@ -12,7 +11,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/route" "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/web/api/v1" "github.com/prometheus/tsdb" "google.golang.org/grpc" @@ -43,6 +41,7 @@ func main() { chunkStoreConfig chunk.StoreConfig distributorConfig distributor.Config + querierConfig querier.Config ingesterConfig ingester.Config configStoreConfig ruler.ConfigStoreConfig rulerConfig ruler.Config @@ -54,7 +53,7 @@ func main() { ) // Ingester needs to know our gRPC listen port. ingesterConfig.ListenPort = &serverConfig.GRPCListenPort - util.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig, + util.RegisterFlags(&serverConfig, &chunkStoreConfig, &distributorConfig, &querierConfig, &ingesterConfig, &configStoreConfig, &rulerConfig, &storageConfig, &schemaConfig, &logLevel) flag.BoolVar(&unauthenticated, "unauthenticated", false, "Set to true to disable multitenancy.") flag.Parse() @@ -124,13 +123,15 @@ func main() { tableManager.Start() defer tableManager.Stop() + engine, queryable := querier.NewEngine(dist, chunkStore, nil, querierConfig.MaxConcurrent, querierConfig.Timeout) + if configStoreConfig.ConfigsAPIURL.String() != "" || configStoreConfig.DBConfig.URI != "" { rulesAPI, err := ruler.NewRulesAPI(configStoreConfig) if err != nil { level.Error(util.Logger).Log("msg", "error initializing ruler config store", "err", err) os.Exit(1) } - rlr, err := ruler.NewRuler(rulerConfig, dist, chunkStore) + rlr, err := ruler.NewRuler(rulerConfig, engine, queryable, dist) if err != nil { level.Error(util.Logger).Log("msg", "error initializing ruler", "err", err) os.Exit(1) @@ -148,9 +149,6 @@ func main() { sampleQueryable := querier.NewQueryable(dist, chunkStore, false) metadataQueryable := querier.NewQueryable(dist, chunkStore, true) - maxConcurrent := 20 - timeout := 2 * time.Minute - engine := promql.NewEngine(util.Logger, nil, maxConcurrent, timeout) api := v1.NewAPI( engine, metadataQueryable, diff --git a/cmd/querier/main.go b/cmd/querier/main.go index f0f0890af1..f1a1472e9f 100644 --- a/cmd/querier/main.go +++ b/cmd/querier/main.go @@ -4,7 +4,6 @@ import ( "flag" "net/http" "os" - "time" "google.golang.org/grpc" @@ -37,12 +36,13 @@ func main() { } ringConfig ring.Config distributorConfig distributor.Config + querierConfig querier.Config chunkStoreConfig chunk.StoreConfig schemaConfig chunk.SchemaConfig storageConfig storage.Config logLevel util.LogLevel ) - util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, + util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &querierConfig, &chunkStoreConfig, &schemaConfig, &storageConfig, &logLevel) flag.Parse() @@ -91,9 +91,7 @@ func main() { sampleQueryable := querier.NewQueryable(dist, chunkStore, false) metadataQueryable := querier.NewQueryable(dist, chunkStore, true) - maxConcurrent := 20 - timeout := 2 * time.Minute - engine := promql.NewEngine(util.Logger, nil, maxConcurrent, timeout) + engine := promql.NewEngine(util.Logger, nil, querierConfig.MaxConcurrent, querierConfig.Timeout) api := v1.NewAPI( engine, metadataQueryable, diff --git a/cmd/ruler/main.go b/cmd/ruler/main.go index e4ff2aaca6..489cfdb781 100644 --- a/cmd/ruler/main.go +++ b/cmd/ruler/main.go @@ -14,6 +14,7 @@ import ( "github.com/weaveworks/cortex/pkg/chunk" "github.com/weaveworks/cortex/pkg/chunk/storage" "github.com/weaveworks/cortex/pkg/distributor" + "github.com/weaveworks/cortex/pkg/querier" "github.com/weaveworks/cortex/pkg/ring" "github.com/weaveworks/cortex/pkg/ruler" "github.com/weaveworks/cortex/pkg/util" @@ -29,6 +30,7 @@ func main() { } ringConfig ring.Config distributorConfig distributor.Config + querierConfig querier.Config rulerConfig ruler.Config chunkStoreConfig chunk.StoreConfig schemaConfig chunk.SchemaConfig @@ -41,7 +43,7 @@ func main() { trace := tracing.NewFromEnv("ruler") defer trace.Close() - util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, + util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &querierConfig, &rulerConfig, &chunkStoreConfig, &storageConfig, &schemaConfig, &configStoreConfig, &logLevel) flag.Parse() @@ -75,7 +77,8 @@ func main() { defer dist.Stop() prometheus.MustRegister(dist) - rlr, err := ruler.NewRuler(rulerConfig, dist, chunkStore) + engine, queryable := querier.NewEngine(dist, chunkStore, prometheus.DefaultRegisterer, querierConfig.MaxConcurrent, querierConfig.Timeout) + rlr, err := ruler.NewRuler(rulerConfig, engine, queryable, dist) if err != nil { level.Error(util.Logger).Log("msg", "error initializing ruler", "err", err) os.Exit(1) diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 414d4b98d1..8d26998fb9 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -2,6 +2,7 @@ package querier import ( "context" + "flag" "net/http" "time" @@ -18,6 +19,18 @@ import ( "github.com/weaveworks/cortex/pkg/util" ) +// Config contains the configuration require to create a querier +type Config struct { + MaxConcurrent int + Timeout time.Duration +} + +// RegisterFlags adds the flags required to config this to the given FlagSet +func (cfg *Config) RegisterFlags(f *flag.FlagSet) { + flag.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") + flag.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") +} + // ChunkStore is the interface we need to get chunks type ChunkStore interface { Get(ctx context.Context, from, through model.Time, matchers ...*labels.Matcher) (model.Matrix, error) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index 2ac1b201af..06d31dff84 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -33,10 +33,8 @@ import ( "github.com/weaveworks/common/instrument" "github.com/weaveworks/common/user" - "github.com/weaveworks/cortex/pkg/chunk" "github.com/weaveworks/cortex/pkg/configs" "github.com/weaveworks/cortex/pkg/distributor" - "github.com/weaveworks/cortex/pkg/querier" "github.com/weaveworks/cortex/pkg/util" ) @@ -198,12 +196,11 @@ func (rn *rulerNotifier) stop() { } // NewRuler creates a new ruler from a distributor and chunk store. -func NewRuler(cfg Config, d *distributor.Distributor, c *chunk.Store) (*Ruler, error) { +func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d *distributor.Distributor) (*Ruler, error) { ncfg, err := buildNotifierConfig(&cfg) if err != nil { return nil, err } - engine, queryable := querier.NewEngine(d, c, prometheus.DefaultRegisterer, cfg.NumWorkers, cfg.GroupTimeout) return &Ruler{ engine: engine, queryable: queryable, diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index 8b5f3c83c5..342fdd5c43 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -13,6 +13,7 @@ import ( "github.com/prometheus/prometheus/notifier" "github.com/stretchr/testify/assert" "github.com/weaveworks/common/user" + "github.com/weaveworks/cortex/pkg/querier" ) func newTestRuler(t *testing.T, alertmanagerURL string) *Ruler { @@ -25,7 +26,9 @@ func newTestRuler(t *testing.T, alertmanagerURL string) *Ruler { // TODO: Populate distributor and chunk store arguments to enable // other kinds of tests. - ruler, err := NewRuler(cfg, nil, nil) + + engine, queryable := querier.NewEngine(nil, nil, nil, 20, 2*time.Minute) + ruler, err := NewRuler(cfg, engine, queryable, nil) if err != nil { t.Fatal(err) } From 1319a274646db1631a6b49c965ec3496b56a675a Mon Sep 17 00:00:00 2001 From: Aaron Kirkbride Date: Fri, 6 Apr 2018 14:14:17 +0100 Subject: [PATCH 7/9] Reuse ruler config for ruler querier engine --- cmd/ruler/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/ruler/main.go b/cmd/ruler/main.go index 489cfdb781..57c2c78a89 100644 --- a/cmd/ruler/main.go +++ b/cmd/ruler/main.go @@ -30,7 +30,6 @@ func main() { } ringConfig ring.Config distributorConfig distributor.Config - querierConfig querier.Config rulerConfig ruler.Config chunkStoreConfig chunk.StoreConfig schemaConfig chunk.SchemaConfig @@ -43,7 +42,7 @@ func main() { trace := tracing.NewFromEnv("ruler") defer trace.Close() - util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &querierConfig, + util.RegisterFlags(&serverConfig, &ringConfig, &distributorConfig, &rulerConfig, &chunkStoreConfig, &storageConfig, &schemaConfig, &configStoreConfig, &logLevel) flag.Parse() @@ -77,7 +76,7 @@ func main() { defer dist.Stop() prometheus.MustRegister(dist) - engine, queryable := querier.NewEngine(dist, chunkStore, prometheus.DefaultRegisterer, querierConfig.MaxConcurrent, querierConfig.Timeout) + engine, queryable := querier.NewEngine(dist, chunkStore, prometheus.DefaultRegisterer, rulerConfig.NumWorkers, rulerConfig.GroupTimeout) rlr, err := ruler.NewRuler(rulerConfig, engine, queryable, dist) if err != nil { level.Error(util.Logger).Log("msg", "error initializing ruler", "err", err) From 746f903d311f553d036b2bcbbb9e0c9c9b2fdef2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Thu, 5 Apr 2018 15:07:12 +0000 Subject: [PATCH 8/9] Clarify passing no flags map to NewEngine() --- cmd/lite/main.go | 2 +- cmd/querier/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lite/main.go b/cmd/lite/main.go index 9269fe7729..a42fcd74fc 100644 --- a/cmd/lite/main.go +++ b/cmd/lite/main.go @@ -155,7 +155,7 @@ func main() { querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{}, func() config.Config { return config.Config{} }, - map[string]string{}, // TODO: include configuration flags + nil, // flags to be served via http; we don't need this func(f http.HandlerFunc) http.HandlerFunc { return f }, func() *tsdb.DB { return nil }, // Only needed for admin APIs. false, // Disable admin APIs. diff --git a/cmd/querier/main.go b/cmd/querier/main.go index f1a1472e9f..334b034a7d 100644 --- a/cmd/querier/main.go +++ b/cmd/querier/main.go @@ -98,7 +98,7 @@ func main() { querier.DummyTargetRetriever{}, querier.DummyAlertmanagerRetriever{}, func() config.Config { return config.Config{} }, - map[string]string{}, // TODO: include configuration flags + nil, // flags to be served via http; we don't need this func(f http.HandlerFunc) http.HandlerFunc { return f }, func() *tsdb.DB { return nil }, // Only needed for admin APIs. false, // Disable admin APIs. From 19d13c739113580a432ce135991ea223b2152c33 Mon Sep 17 00:00:00 2001 From: Aaron Kirkbride Date: Fri, 6 Apr 2018 15:59:03 +0100 Subject: [PATCH 9/9] Update Gopkg.lock --- Gopkg.lock | 2 +- .../github.com/json-iterator/go/LICENSE~HEAD | 21 ------------------- .../go/LICENSE~Update prometheus to latest | 21 ------------------- 3 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 vendor/github.com/json-iterator/go/LICENSE~HEAD delete mode 100644 vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest diff --git a/Gopkg.lock b/Gopkg.lock index 4c05f0f57f..84b8dd7014 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1190,6 +1190,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a45496e76b9d180c0d63003459b5cd153c8bc0c430ea97e6f8e3902fb3c9ee8c" + inputs-digest = "fc2b611011b916790e0e1f112a7e0359692b3963b08ca30fbdfc34c37278246c" solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/json-iterator/go/LICENSE~HEAD b/vendor/github.com/json-iterator/go/LICENSE~HEAD deleted file mode 100644 index 2cf4f5ab28..0000000000 --- a/vendor/github.com/json-iterator/go/LICENSE~HEAD +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 json-iterator - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest b/vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest deleted file mode 100644 index 2cf4f5ab28..0000000000 --- a/vendor/github.com/json-iterator/go/LICENSE~Update prometheus to latest +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2016 json-iterator - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE.