More visibility on ingesters

distributor.go

@@ -31,7 +31,7 @@ import (
 
 var (
 	numClientsDesc = prometheus.NewDesc(
-		"prometheus_distributor_ingester_clients",
+		"prism_distributor_ingester_clients",
 		"The current number of ingester clients.",
 		nil, nil,
 	)
@@ -51,6 +51,7 @@ type Distributor struct {
 	ingesterAppendFailures *prometheus.CounterVec
 	ingesterQueries        *prometheus.CounterVec
 	ingesterQueryFailures  *prometheus.CounterVec
+	ingestersAlive         *prometheus.Desc
 }
 
 // ReadRing represents the read inferface to the ring.
@@ -114,7 +115,7 @@ func NewDistributor(cfg DistributorConfig) (*Distributor, error) {
 		}, []string{"ingester"}),
 		ingesterAppendFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Namespace: "prism",
-			Name:      "distributor_ingester_appends_total",
+			Name:      "distributor_ingester_append_failures_total",
 			Help:      "The total number of failed batch appends sent to ingesters.",
 		}, []string{"ingester"}),
 		ingesterQueries: prometheus.NewCounterVec(prometheus.CounterOpts{
@@ -124,9 +125,14 @@ func NewDistributor(cfg DistributorConfig) (*Distributor, error) {
 		}, []string{"ingester"}),
 		ingesterQueryFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Namespace: "prism",
-			Name:      "distributor_ingester_appends_total",
+			Name:      "distributor_ingester_query_failures_total",
 			Help:      "The total number of failed queries sent to ingesters.",
 		}, []string{"ingester"}),
+		ingestersAlive: prometheus.NewDesc(
+			"prism_distributor_ingesters_alive_total",
+			"Number of ingesters in the ring that have heartbeats within timeout.",
+			nil, nil,
+		),
 	}, nil
 }
 
@@ -257,6 +263,10 @@ func (d *Distributor) Query(ctx context.Context, from, to model.Time, matchers .
 			return err
 		}
 
+		if len(ingesters) < d.cfg.MinReadSuccesses {
+			return fmt.Errorf("Could only find %d ingesters for query. Need at least %d", len(ingesters), d.cfg.MinReadSuccesses)
+		}
+
 		// Fetch samples from multiple ingesters and group them by fingerprint (unsorted
 		// and with overlap).
 		successes := 0
@@ -363,6 +373,11 @@ func (d *Distributor) Describe(ch chan<- *prometheus.Desc) {
 	d.sendDuration.Describe(ch)
 	d.cfg.Ring.Describe(ch)
 	ch <- numClientsDesc
+	d.ingesterAppends.Describe(ch)
+	d.ingesterAppendFailures.Describe(ch)
+	d.ingesterQueries.Describe(ch)
+	d.ingesterQueryFailures.Describe(ch)
+	ch <- d.ingestersAlive
 }
 
 // Collect implements prometheus.Collector.
@@ -371,6 +386,16 @@ func (d *Distributor) Collect(ch chan<- prometheus.Metric) {
 	ch <- d.receivedSamples
 	d.sendDuration.Collect(ch)
 	d.cfg.Ring.Collect(ch)
+	d.ingesterAppends.Collect(ch)
+	d.ingesterAppendFailures.Collect(ch)
+	d.ingesterQueries.Collect(ch)
+	d.ingesterQueryFailures.Collect(ch)
+
+	ch <- prometheus.MustNewConstMetric(
+		d.ingestersAlive,
+		prometheus.GaugeValue,
+		float64(len(d.cfg.Ring.GetAll(d.cfg.HeartbeatTimeout))),
+	)
 
 	d.clientsMtx.RLock()
 	defer d.clientsMtx.RUnlock()

ring/ring.go

@@ -50,8 +50,8 @@ type Ring struct {
 	ringDesc Desc
 
 	ingesterOwnershipDesc *prometheus.Desc
-	ingesterTotalDesc     *prometheus.Desc
-	tokensTotalDesc       *prometheus.Desc
+	numIngestersDesc      *prometheus.Desc
+	numTokensDesc         *prometheus.Desc
 }
 
 // New creates a new Ring
@@ -61,17 +61,17 @@ func New(client CoordinationStateClient) *Ring {
 		quit:   make(chan struct{}),
 		done:   make(chan struct{}),
 		ingesterOwnershipDesc: prometheus.NewDesc(
-			"prometheus_distributor_ingester_ownership_percent",
+			"prism_distributor_ingester_ownership_percent",
 			"The percent ownership of the ring by ingester",
 			[]string{"ingester"}, nil,
 		),
-		ingesterTotalDesc: prometheus.NewDesc(
-			"prometheus_distributor_ingesters_total",
+		numIngestersDesc: prometheus.NewDesc(
+			"prism_distributor_ingesters",
 			"Number of ingesters in the ring",
 			nil, nil,
 		),
-		tokensTotalDesc: prometheus.NewDesc(
-			"prometheus_distributor_tokens_total",
+		numTokensDesc: prometheus.NewDesc(
+			"prism_distributor_tokens",
 			"Number of tokens in the ring",
 			nil, nil,
 		),
@@ -166,8 +166,8 @@ func (r *Ring) search(key uint32) int {
 // Describe implements prometheus.Collector.
 func (r *Ring) Describe(ch chan<- *prometheus.Desc) {
 	ch <- r.ingesterOwnershipDesc
-	ch <- r.ingesterTotalDesc
-	ch <- r.tokensTotalDesc
+	ch <- r.numIngestersDesc
+	ch <- r.numTokensDesc
 }
 
 // Collect implements prometheus.Collector.
@@ -196,12 +196,12 @@ func (r *Ring) Collect(ch chan<- prometheus.Metric) {
 	}
 
 	ch <- prometheus.MustNewConstMetric(
-		r.ingesterTotalDesc,
+		r.numIngestersDesc,
 		prometheus.GaugeValue,
 		float64(len(r.ringDesc.Ingesters)),
 	)
 	ch <- prometheus.MustNewConstMetric(
-		r.tokensTotalDesc,
+		r.numTokensDesc,
 		prometheus.GaugeValue,
 		float64(len(r.ringDesc.Tokens)),
 	)