Merge pull request #83 from weaveworks/tune-historgram-buckets

Tune histogram bucket sizes

Author: tomwilkie

chunk/chunk_cache.go

@@ -39,7 +39,8 @@ var (
 		Namespace: "cortex",
 		Name:      "memcache_request_duration_seconds",
 		Help:      "Total time spent in seconds doing memcache requests.",
-		Buckets:   []float64{.001, .0025, .005, .01, .025, .05},
+		// Memecache requests are very quick: smallest bucket is 16us, biggest is 1s
+		Buckets: prometheus.ExponentialBuckets(0.000016, 4, 8),
 	}, []string{"method", "status_code"})
 )
 

chunk/chunk_store.go

@@ -54,7 +54,10 @@ var (
 		Namespace: "cortex",
 		Name:      "dynamo_request_duration_seconds",
 		Help:      "Time spent doing DynamoDB requests.",
-		Buckets:   []float64{.001, .0025, .005, .01, .025, .05, .1, .25, .5, 1},
+
+		// DynamoDB latency seems to range from a few ms to a few sec and is
+		// important.  So use 8 buckets from 64us to 8s.
+		Buckets: prometheus.ExponentialBuckets(0.000128, 4, 8),
 	}, []string{"operation", "status_code"})
 	dynamoRequestPages = prometheus.NewHistogram(prometheus.HistogramOpts{
 		Namespace: "cortex",

cmd/cortex/main.go

@@ -53,7 +53,12 @@ var (
 		Namespace: "cortex",
 		Name:      "request_duration_seconds",
 		Help:      "Time (in seconds) spent serving HTTP requests.",
-		Buckets:   prometheus.DefBuckets,
+
+		// Cortex latency can be very low (ingesters typically take a few ms to
+		// process a request), all the way to very high (queries can take tens of
+		// second).  As its important, we use 10 buckets.  Smallest is 128us
+		// biggest is 130s.
+		Buckets: prometheus.ExponentialBuckets(0.000128, 4, 10),
 	}, []string{"method", "route", "status_code", "ws"})
 )