Add metric to track pre-validation native histogram bucket count per user (#7569)

Signed-off-by: Paurush Garg <paurushg@amazon.com>

Author: PaurushGarg

CHANGELOG.md

@@ -28,7 +28,7 @@
 * [ENHANCEMENT] Instrument Ingester CPU profile with source for read APIs. #7494
 * [ENHANCEMENT] Ingester: Convert expanded postings cache from FIFO to LRU eviction to retain frequently-queried entries under memory pressure. #7510
 * [ENHANCEMENT] Querier: Detach series label and chunk data from gRPC unmarshal buffers in store-gateway streaming path, allowing the Go GC to reclaim receive buffers. #7519
-
+* [ENHANCEMENT] Distributor: Added `cortex_distributor_received_histogram_buckets` metric to track number of buckets in received native histogram samples before validation, per user. #7569
 * [ENHANCEMENT] Distributor: Add `WrappedHistogram` with configurable size limit (`-validation.max-native-histogram-size-bytes`) to cap native histogram protobuf size before unmarshalling. #7570
 * [ENHANCEMENT] Ingester: Add lazy regex evaluation on head postings cache miss. Defers expensive regex matchers on high-cardinality labels to per-series filtering when a selective equality matcher already narrows the result set. Configured via `-blocks-storage.expanded_postings_cache.head.lazy-matcher-max-cardinality` (disabled by default). #7553
 * [BUGFIX] Querier: Fix queryWithRetry and labelsWithRetry returning (nil, nil) on cancelled context by propagating ctx.Err(). #7370
@@ -47,6 +47,7 @@
 * [BUGFIX] Ingester: Release the TSDB appender on every early-return path in `Push` (e.g. out-of-order label set) by deferring `Rollback`. Previously such requests leaked TSDB head series references, mmap'd chunks and pending state per request, causing the `cortex_ingester_tsdb_head_active_appenders` gauge to grow unbounded. #7528
 * [BUGFIX] Ring: Fix ring token conflict resolution only applied to updated instance and make constantly token conflict check during instance observe period.
 * [BUGFIX] Distributor: Fix a panic (`slice bounds out of range`) in the stream push path when the context deadline expires while the worker goroutine is still marshalling a `WriteRequest`. #7541
+* [BUGFIX] Query Frontend: Fix native histogram responses not being handled correctly in `minTime()` sort ordering for split_by_interval merge. #7555
 
 ## 1.21.0 2026-04-24
 

pkg/distributor/distributor.go

@@ -124,6 +124,7 @@ type Distributor struct {
 	incomingMetadata                 *prometheus.CounterVec
 	nonHASamples                     *prometheus.CounterVec
 	dedupedSamples                   *prometheus.CounterVec
+	receivedHistogramBuckets         *prometheus.HistogramVec
 	labelsHistogram                  prometheus.Histogram
 	ingesterAppends                  *prometheus.CounterVec
 	ingesterAppendFailures           *prometheus.CounterVec
@@ -353,6 +354,15 @@ func New(cfg Config, clientConfig ingester_client.Config, limits *validation.Ove
 			Name:      "distributor_received_metadata_total",
 			Help:      "The total number of received metadata, excluding rejected.",
 		}, []string{"user"}),
+		receivedHistogramBuckets: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{
+			Namespace:                       "cortex",
+			Name:                            "distributor_received_histogram_buckets",
+			Help:                            "The number of buckets in received native histogram samples before validation, per user.",
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
+			Buckets:                         prometheus.ExponentialBuckets(1, 2, 10), // 1 to 512 buckets
+		}, []string{"user"}),
 		incomingSamples: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Namespace: "cortex",
 			Name:      "distributor_samples_in_total",
@@ -530,6 +540,7 @@ func (d *Distributor) cleanupInactiveUser(userID string) {
 	d.receivedSamples.DeleteLabelValues(userID, sampleMetricTypeHistogramNHCB)
 	d.receivedExemplars.DeleteLabelValues(userID)
 	d.receivedMetadata.DeleteLabelValues(userID)
+	d.receivedHistogramBuckets.DeleteLabelValues(userID)
 	d.incomingSamples.DeleteLabelValues(userID, sampleMetricTypeFloat)
 	d.incomingSamples.DeleteLabelValues(userID, sampleMetricTypeHistogram)
 	d.incomingSamples.DeleteLabelValues(userID, sampleMetricTypeHistogramNHCB)
@@ -691,10 +702,12 @@ func (d *Distributor) validateSeries(ts cortexpb.PreallocTimeseries, userID stri
 	if len(ts.Histograms) > 0 {
 		// Only alloc when data present
 		histograms = make([]cortexpb.WrappedHistogram, 0, len(ts.Histograms))
+		receivedBucketsObserver := d.receivedHistogramBuckets.WithLabelValues(userID)
 		for i, h := range ts.Histograms {
 			if err := validation.ValidateSampleTimestamp(d.validateMetrics, limits, userID, ts.Labels, h.TimestampMs); err != nil {
 				return emptyPreallocSeries, err
 			}
+			receivedBucketsObserver.Observe(float64(h.BucketCount()))
 			convertedHistogram, err := validation.ValidateNativeHistogram(d.validateMetrics, limits, userID, ts.Labels, h.Histogram)
 			if err != nil {
 				return emptyPreallocSeries, err

pkg/distributor/distributor_test.go

@@ -20,6 +20,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
+	dto "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/model/relabel"
@@ -4938,3 +4939,35 @@ func TestDistributor_ShuffleShardingIngestersLookbackPeriod_Validation(t *testin
 		})
 	}
 }
+
+func TestDistributor_ReceivedHistogramBucketsMetric(t *testing.T) {
+	t.Parallel()
+
+	limits := &validation.Limits{}
+	flagext.DefaultValues(limits)
+	// Set a bucket limit so resolution reduction kicks in, but the metric should still capture the original count.
+	limits.MaxNativeHistogramBuckets = 4
+
+	ds, _, _, _ := prepare(t, prepConfig{
+		numIngesters:     3,
+		happyIngesters:   3,
+		numDistributors:  1,
+		shardByAllLabels: true,
+		limits:           limits,
+	})
+
+	// Push a native histogram sample. GenerateTestHistogram produces 4 positive + 4 negative buckets + zero bucket = 9 buckets.
+	ctx := user.InjectOrgID(context.Background(), "user")
+	req := makeWriteRequest(0, 0, 0, 1)
+	_, err := ds[0].Push(ctx, req)
+	require.NoError(t, err)
+
+	// Verify the receivedHistogramBuckets metric observed the pre-validation bucket count.
+	m := &dto.Metric{}
+	observer, err := ds[0].receivedHistogramBuckets.GetMetricWithLabelValues("user")
+	require.NoError(t, err)
+	require.NoError(t, observer.(prometheus.Metric).Write(m))
+	require.Equal(t, uint64(1), m.GetHistogram().GetSampleCount())
+	// GenerateTestHistogram(0): 4 positive + 4 negative + 1 zero = 9 buckets
+	require.Equal(t, float64(9), m.GetHistogram().GetSampleSum())
+}

pkg/ingester/ingester.go

@@ -1527,6 +1527,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 		}
 
 		if i.limits.EnableNativeHistograms(userID) {
+			ingestedBucketsObserver := i.metrics.ingestedHistogramBuckets.WithLabelValues(userID)
 			for _, hp := range ts.Histograms {
 				var (
 					err error
@@ -1551,7 +1552,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 				if ref != 0 {
 					if _, err = app.AppendHistogram(ref, copiedLabels, hp.TimestampMs, h, fh); err == nil {
 						succeededHistogramsCount++
-						i.metrics.ingestedHistogramBuckets.WithLabelValues(userID).Observe(float64(hp.BucketCount()))
+						ingestedBucketsObserver.Observe(float64(hp.BucketCount()))
 						continue
 					}
 				} else {
@@ -1563,7 +1564,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
 							newSeries = append(newSeries, copiedLabels)
 						}
 						succeededHistogramsCount++
-						i.metrics.ingestedHistogramBuckets.WithLabelValues(userID).Observe(float64(hp.BucketCount()))
+						ingestedBucketsObserver.Observe(float64(hp.BucketCount()))
 						continue
 					}
 				}

pkg/ingester/metrics.go

@@ -1,6 +1,8 @@
 package ingester
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -142,7 +144,7 @@ func newIngesterMetrics(r prometheus.Registerer,
 			Help:                            "The number of ingested native histogram buckets per user.",
 			NativeHistogramBucketFactor:     1.1,
 			NativeHistogramMaxBucketNumber:  100,
-			NativeHistogramMinResetDuration: 1,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 			Buckets:                         prometheus.ExponentialBuckets(1, 2, 10), // 1 to 512 buckets
 		}, []string{"user"}),
 		oooLabelsTotal: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
@@ -317,23 +319,23 @@ func newIngesterMetrics(r prometheus.Registerer,
 			Help:                            "Length (in bytes) of unoptimized regex patterns in queries.",
 			NativeHistogramBucketFactor:     1.1,
 			NativeHistogramMaxBucketNumber:  100,
-			NativeHistogramMinResetDuration: 1,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 			Buckets:                         prometheus.ExponentialBuckets(1, 2, 12), // 1 to 4096 bytes
 		})
 		m.unoptimizedRegexLabelCardinality = promauto.With(r).NewHistogram(prometheus.HistogramOpts{
 			Name:                            "cortex_ingester_unoptimized_regex_label_cardinality",
 			Help:                            "Cardinality of labels queried with unoptimized regex matchers.",
 			NativeHistogramBucketFactor:     1.1,
 			NativeHistogramMaxBucketNumber:  100,
-			NativeHistogramMinResetDuration: 1,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 			Buckets:                         prometheus.ExponentialBuckets(1, 4, 10), // 1 to ~1M
 		})
 		m.unoptimizedRegexTotalValueLength = promauto.With(r).NewHistogram(prometheus.HistogramOpts{
 			Name:                            "cortex_ingester_unoptimized_regex_total_value_length_bytes",
 			Help:                            "Total length (in bytes) of all label values for labels queried with unoptimized regex matchers.",
 			NativeHistogramBucketFactor:     1.1,
 			NativeHistogramMaxBucketNumber:  100,
-			NativeHistogramMinResetDuration: 1,
+			NativeHistogramMinResetDuration: 1 * time.Hour,
 			Buckets:                         prometheus.ExponentialBuckets(1, 4, 12), // 1 to ~16M bytes
 		})
 		m.unoptimizedRegexRejectedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{