Comment unsafe memory usage in ingester push path

Signed-off-by: Bryan Boreham <[email protected]>

Author: bboreham

pkg/ingester/index/index.go

@@ -32,9 +32,11 @@ func New() *InvertedIndex {
 }
 
 // Add a fingerprint under the specified labels.
+// NOTE: memory for `labels` is unsafe; anything retained beyond the
+// life of this function must be copied
 func (ii *InvertedIndex) Add(labels []client.LabelAdapter, fp model.Fingerprint) labels.Labels {
 	shard := &ii.shards[util.HashFP(fp)%indexShards]
-	return shard.add(labels, fp)
+	return shard.add(labels, fp) // add() returns 'interned' values so the original labels are not retained
 }
 
 // Lookup all fingerprints for the provided matchers.
@@ -108,7 +110,9 @@ func copyString(s string) string {
 	return string([]byte(s))
 }
 
-// add metric to the index; return all the name/value pairs as strings from the index, sorted
+// add metric to the index; return all the name/value pairs as a fresh
+// sorted slice, referencing 'interned' strings from the index so that
+// no references are retained to the memory of `metric`.
 func (shard *indexShard) add(metric []client.LabelAdapter, fp model.Fingerprint) labels.Labels {
 	shard.mtx.Lock()
 	defer shard.mtx.Unlock()

pkg/ingester/ingester.go

@@ -299,8 +299,11 @@ func (i *Ingester) Push(ctx old_ctx.Context, req *client.WriteRequest) (*client.
 		}
 	}
 
+	// NOTE: because we use `unsafe` in deserialisation, we must not
+	// retain anything from `req` past the call to ReuseSlice
 	for _, ts := range req.Timeseries {
 		for _, s := range ts.Samples {
+			// append() copies the memory in `ts.Labels` except on the error path
 			err := i.append(ctx, userID, ts.Labels, model.Time(s.TimestampMs), model.SampleValue(s.Value), req.Source, record)
 			if err == nil {
 				continue
@@ -312,11 +315,13 @@ func (i *Ingester) Push(ctx old_ctx.Context, req *client.WriteRequest) (*client.
 				continue
 			}
 
+			// non-validation error: abandon this request
 			return nil, wrapWithUser(err, userID)
 		}
 	}
 
 	if lastPartialErr != nil {
+		// WrappedError turns the error into a string so it no longer references `req`
 		return &client.WriteResponse{}, lastPartialErr.WrapWithUser(userID).WrappedError()
 	}
 
@@ -331,6 +336,8 @@ func (i *Ingester) Push(ctx old_ctx.Context, req *client.WriteRequest) (*client.
 	return &client.WriteResponse{}, nil
 }
 
+// NOTE: memory for `labels` is unsafe; anything retained beyond the
+// life of this function must be copied
 func (i *Ingester) append(ctx context.Context, userID string, labels labelPairs, timestamp model.Time, value model.SampleValue, source client.WriteRequest_SourceEnum, record *Record) error {
 	labels.removeBlanks()
 
@@ -349,6 +356,7 @@ func (i *Ingester) append(ctx context.Context, userID string, labels labelPairs,
 		return fmt.Errorf("ingester stopping")
 	}
 
+	// getOrCreateSeries copies the memory for `labels`, except on the error path.
 	state, fp, series, err := i.userStates.getOrCreateSeries(ctx, userID, labels, record)
 	if err != nil {
 		if ve, ok := err.(*validationError); ok {

pkg/ingester/user_state.go

@@ -155,12 +155,17 @@ func (us *userStates) getViaContext(ctx context.Context) (*userState, bool, erro
 	return state, ok, nil
 }
 
+// NOTE: memory for `labels` is unsafe; anything retained beyond the
+// life of this function must be copied
 func (us *userStates) getOrCreateSeries(ctx context.Context, userID string, labels []client.LabelAdapter, record *Record) (*userState, model.Fingerprint, *memorySeries, error) {
 	state := us.getOrCreate(userID)
+	// WARNING: `err` may have a reference to unsafe memory in `labels`
 	fp, series, err := state.getSeries(labels, record)
 	return state, fp, series, err
 }
 
+// NOTE: memory for `metric` is unsafe; anything retained beyond the
+// life of this function must be copied
 func (u *userState) getSeries(metric labelPairs, record *Record) (model.Fingerprint, *memorySeries, error) {
 	rawFP := client.FastFingerprint(metric)
 	u.fpLocker.Lock(rawFP)
@@ -197,6 +202,7 @@ func (u *userState) createSeriesWithFingerprint(fp model.Fingerprint, metric lab
 		}
 	}
 
+	// MetricNameFromLabelAdapters returns a copy of the string in `metric`
 	metricName, err := extract.MetricNameFromLabelAdapters(metric)
 	if err != nil {
 		return nil, err
@@ -205,6 +211,7 @@ func (u *userState) createSeriesWithFingerprint(fp model.Fingerprint, metric lab
 	if !recovery {
 		// Check if the per-metric limit has been exceeded
 		if err = u.canAddSeriesFor(string(metricName)); err != nil {
+			// WARNING: returns a reference to `metric`
 			return nil, makeMetricLimitError(perMetricSeriesLimit, client.FromLabelAdaptersToLabels(metric), err)
 		}
 	}
@@ -219,7 +226,7 @@ func (u *userState) createSeriesWithFingerprint(fp model.Fingerprint, metric lab
 		})
 	}
 
-	labels := u.index.Add(metric, fp)
+	labels := u.index.Add(metric, fp) // Add() returns 'interned' values so the original labels are not retained
 	series := newMemorySeries(labels)
 	u.fpToSeries.put(fp, series)