Spread the sync load over time (#1587)

* Spread the sync load over time

Signed-off-by: Goutham Veeramachaneni <[email protected]>

* Move seed to main

Signed-off-by: Goutham Veeramachaneni <[email protected]>

Author: gouthamve

cmd/cortex/main.go

@@ -4,8 +4,10 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"runtime"
+	"time"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
@@ -57,6 +59,9 @@ func main() {
 	trace := tracing.NewFromEnv("cortex-" + cfg.Target.String())
 	defer trace.Close()
 
+	// Initialise seed for randomness usage.
+	rand.Seed(time.Now().UnixNano())
+
 	t, err := cortex.New(cfg)
 	util.CheckFatal("initializing cortex", err)
 

pkg/chunk/table_manager.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"math/rand"
 	"sort"
 	"strings"
 	"sync"
@@ -166,6 +167,9 @@ func (m *TableManager) Stop() {
 func (m *TableManager) loop() {
 	defer m.wait.Done()
 
+	// Sleep for a bit to spread the sync load across different times if the tablemanagers are all started at once.
+	time.Sleep(time.Duration(rand.Int63n(int64(m.cfg.DynamoDBPollInterval))))
+
 	ticker := time.NewTicker(m.cfg.DynamoDBPollInterval)
 	defer ticker.Stop()