Skip to content

Commit df9af3a

Browse files
aknuds1aknuds1
authored
Use backoff functionality from github.com/grafana/dskit (#4426)
* Use dskit/backoff Signed-off-by: Arve Knudsen <[email protected]> * Fix formatting Signed-off-by: Arve Knudsen <[email protected]> * Remove changelog entry Signed-off-by: Arve Knudsen <[email protected]>
1 parent 35919e4 commit df9af3a

File tree

147 files changed

+3249
-1612
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

147 files changed

+3249
-1612
lines changed

‎go.mod

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ require (
1212
github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15
1313
github.com/alicebob/miniredis/v2 v2.14.3
1414
github.com/armon/go-metrics v0.3.6
15-
github.com/aws/aws-sdk-go v1.38.60
15+
github.com/aws/aws-sdk-go v1.38.68
1616
github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
1717
github.com/cespare/xxhash v1.1.0
1818
github.com/dustin/go-humanize v1.0.0
1919
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
2020
github.com/felixge/fgprof v0.9.1
2121
github.com/fsouza/fake-gcs-server v1.7.0
22-
github.com/go-kit/kit v0.10.0
22+
github.com/go-kit/kit v0.11.0
2323
github.com/go-openapi/strfmt v0.20.1
2424
github.com/go-openapi/swag v0.19.15
2525
github.com/go-redis/redis/v8 v8.9.0
@@ -30,6 +30,7 @@ require (
3030
github.com/golang/protobuf v1.5.2
3131
github.com/golang/snappy v0.0.3
3232
github.com/gorilla/mux v1.7.3
33+
github.com/grafana/dskit v0.0.0-20210817085554-1b69d2de136f
3334
github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
3435
github.com/hashicorp/consul/api v1.8.1
3536
github.com/hashicorp/go-cleanhttp v0.5.1
@@ -59,9 +60,9 @@ require (
5960
github.com/weaveworks/common v0.0.0-20210419092856-009d1eebd624
6061
go.etcd.io/bbolt v1.3.5
6162
go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489
62-
go.etcd.io/etcd/client/v3 v3.5.0-alpha.0.0.20210225194612-fa82d11a958a
63+
go.etcd.io/etcd/client/v3 v3.5.0
6364
go.etcd.io/etcd/server/v3 v3.5.0-alpha.0.0.20210225194612-fa82d11a958a
64-
go.uber.org/atomic v1.8.0
65+
go.uber.org/atomic v1.9.0
6566
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b
6667
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
6768
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6

‎go.sum

Lines changed: 47 additions & 14 deletions
Large diffs are not rendered by default.

‎integration/e2e/composite_service.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@ import (
55
"fmt"
66
"time"
77

8+
"github.com/grafana/dskit/backoff"
89
"github.com/pkg/errors"
9-
10-
"github.com/cortexproject/cortex/pkg/util"
1110
)
1211

1312
// CompositeHTTPService abstract an higher-level service composed, under the hood,
@@ -16,13 +15,13 @@ type CompositeHTTPService struct {
1615
services []*HTTPService
1716

1817
// Generic retry backoff.
19-
retryBackoff *util.Backoff
18+
retryBackoff *backoff.Backoff
2019
}
2120

2221
func NewCompositeHTTPService(services ...*HTTPService) *CompositeHTTPService {
2322
return &CompositeHTTPService{
2423
services: services,
25-
retryBackoff: util.NewBackoff(context.Background(), util.BackoffConfig{
24+
retryBackoff: backoff.New(context.Background(), backoff.Config{
2625
MinBackoff: 300 * time.Millisecond,
2726
MaxBackoff: 600 * time.Millisecond,
2827
MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯

‎integration/e2e/service.go

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,10 @@ import (
1313
"time"
1414

1515
"github.com/go-kit/kit/log"
16+
"github.com/grafana/dskit/backoff"
1617
"github.com/pkg/errors"
1718
"github.com/prometheus/common/expfmt"
1819
"github.com/thanos-io/thanos/pkg/runutil"
19-
20-
"github.com/cortexproject/cortex/pkg/util"
2120
)
2221

2322
var (
@@ -43,7 +42,7 @@ type ConcreteService struct {
4342
networkPortsContainerToLocal map[int]int
4443

4544
// Generic retry backoff.
46-
retryBackoff *util.Backoff
45+
retryBackoff *backoff.Backoff
4746

4847
// docker NetworkName used to start this container.
4948
// If empty it means service is stopped.
@@ -64,7 +63,7 @@ func NewConcreteService(
6463
command: command,
6564
networkPortsContainerToLocal: map[int]int{},
6665
readiness: readiness,
67-
retryBackoff: util.NewBackoff(context.Background(), util.BackoffConfig{
66+
retryBackoff: backoff.New(context.Background(), backoff.Config{
6867
MinBackoff: 300 * time.Millisecond,
6968
MaxBackoff: 600 * time.Millisecond,
7069
MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯
@@ -80,8 +79,8 @@ func (s *ConcreteService) Name() string { return s.name }
8079

8180
// Less often used options.
8281

83-
func (s *ConcreteService) SetBackoff(cfg util.BackoffConfig) {
84-
s.retryBackoff = util.NewBackoff(context.Background(), cfg)
82+
func (s *ConcreteService) SetBackoff(cfg backoff.Config) {
83+
s.retryBackoff = backoff.New(context.Background(), cfg)
8584
}
8685

8786
func (s *ConcreteService) SetEnvVars(env map[string]string) {

‎integration/e2e/service_test.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,9 @@ import (
1010
"testing"
1111
"time"
1212

13+
"github.com/grafana/dskit/backoff"
1314
"github.com/stretchr/testify/assert"
1415
"github.com/stretchr/testify/require"
15-
16-
"github.com/cortexproject/cortex/pkg/util"
1716
)
1817

1918
func TestWaitSumMetric(t *testing.T) {
@@ -80,15 +79,15 @@ metric_b_summary_count 1
8079
},
8180
}
8281

83-
s.SetBackoff(util.BackoffConfig{
82+
s.SetBackoff(backoff.Config{
8483
MinBackoff: 300 * time.Millisecond,
8584
MaxBackoff: 600 * time.Millisecond,
8685
MaxRetries: 50,
8786
})
8887
require.NoError(t, s.WaitSumMetrics(Equals(221), "metric_a"))
8988

9089
// No retry.
91-
s.SetBackoff(util.BackoffConfig{
90+
s.SetBackoff(backoff.Config{
9291
MinBackoff: 0,
9392
MaxBackoff: 0,
9493
MaxRetries: 1,
@@ -164,7 +163,7 @@ metric_b 1000
164163
},
165164
}
166165

167-
s.SetBackoff(util.BackoffConfig{
166+
s.SetBackoff(backoff.Config{
168167
MinBackoff: 300 * time.Millisecond,
169168
MaxBackoff: 600 * time.Millisecond,
170169
MaxRetries: 50,

‎integration/integration_memberlist_single_binary_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@ import (
1111
"testing"
1212
"time"
1313

14+
"github.com/grafana/dskit/backoff"
1415
"github.com/stretchr/testify/require"
1516
"golang.org/x/sync/errgroup"
1617

1718
"github.com/cortexproject/cortex/integration/ca"
1819
"github.com/cortexproject/cortex/integration/e2e"
1920
e2edb "github.com/cortexproject/cortex/integration/e2e/db"
2021
"github.com/cortexproject/cortex/integration/e2ecortex"
21-
"github.com/cortexproject/cortex/pkg/util"
2222
)
2323

2424
func TestSingleBinaryWithMemberlist(t *testing.T) {
@@ -145,7 +145,7 @@ func newSingleBinary(name string, servername string, join string, testFlags map[
145145
8000,
146146
)
147147

148-
backOff := util.BackoffConfig{
148+
backOff := backoff.Config{
149149
MinBackoff: 200 * time.Millisecond,
150150
MaxBackoff: 500 * time.Millisecond, // Bump max backoff... things take little longer with memberlist.
151151
MaxRetries: 100,

‎pkg/chunk/aws/dynamodb_storage_client.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"time"
1212

1313
"github.com/go-kit/kit/log/level"
14+
"github.com/grafana/dskit/backoff"
1415
ot "github.com/opentracing/opentracing-go"
1516
otlog "github.com/opentracing/opentracing-go/log"
1617
"golang.org/x/time/rate"
@@ -60,7 +61,7 @@ type DynamoDBConfig struct {
6061
Metrics MetricsAutoScalingConfig `yaml:"metrics"`
6162
ChunkGangSize int `yaml:"chunk_gang_size"`
6263
ChunkGetMaxParallelism int `yaml:"chunk_get_max_parallelism"`
63-
BackoffConfig util.BackoffConfig `yaml:"backoff_config"`
64+
BackoffConfig backoff.Config `yaml:"backoff_config"`
6465
}
6566

6667
// RegisterFlags adds the flags required to config this to the given FlagSet
@@ -177,7 +178,7 @@ func (a dynamoDBStorageClient) BatchWrite(ctx context.Context, input chunk.Write
177178
outstanding := input.(dynamoDBWriteBatch)
178179
unprocessed := dynamoDBWriteBatch{}
179180

180-
backoff := util.NewBackoff(ctx, a.cfg.BackoffConfig)
181+
backoff := backoff.New(ctx, a.cfg.BackoffConfig)
181182

182183
for outstanding.Len()+unprocessed.Len() > 0 && backoff.Ongoing() {
183184
requests := dynamoDBWriteBatch{}
@@ -433,7 +434,7 @@ func (a dynamoDBStorageClient) getDynamoDBChunks(ctx context.Context, chunks []c
433434

434435
result := []chunk.Chunk{}
435436
unprocessed := dynamoDBReadRequest{}
436-
backoff := util.NewBackoff(ctx, a.cfg.BackoffConfig)
437+
backoff := backoff.New(ctx, a.cfg.BackoffConfig)
437438

438439
for outstanding.Len()+unprocessed.Len() > 0 && backoff.Ongoing() {
439440
requests := dynamoDBReadRequest{}

‎pkg/chunk/aws/dynamodb_table_client.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@ import (
99
"github.com/aws/aws-sdk-go/service/dynamodb"
1010
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface"
1111
"github.com/go-kit/kit/log/level"
12+
"github.com/grafana/dskit/backoff"
1213
"github.com/pkg/errors"
1314
"github.com/prometheus/client_golang/prometheus"
1415
"github.com/weaveworks/common/instrument"
1516
"golang.org/x/time/rate"
1617

1718
"github.com/cortexproject/cortex/pkg/chunk"
18-
"github.com/cortexproject/cortex/pkg/util"
1919
"github.com/cortexproject/cortex/pkg/util/log"
2020
)
2121

@@ -30,7 +30,7 @@ type autoscale interface {
3030

3131
type callManager struct {
3232
limiter *rate.Limiter
33-
backoffConfig util.BackoffConfig
33+
backoffConfig backoff.Config
3434
}
3535

3636
type dynamoTableClient struct {
@@ -80,7 +80,7 @@ func (d callManager) backoffAndRetry(ctx context.Context, fn func(context.Contex
8080
_ = d.limiter.Wait(ctx)
8181
}
8282

83-
backoff := util.NewBackoff(ctx, d.backoffConfig)
83+
backoff := backoff.New(ctx, d.backoffConfig)
8484
for backoff.Ongoing() {
8585
if err := fn(ctx); err != nil {
8686
if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "ThrottlingException" {

‎pkg/chunk/aws/fixtures.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,12 @@ import (
55
"io"
66
"time"
77

8+
"github.com/grafana/dskit/backoff"
89
"golang.org/x/time/rate"
910

1011
"github.com/cortexproject/cortex/pkg/chunk"
1112
"github.com/cortexproject/cortex/pkg/chunk/objectclient"
1213
"github.com/cortexproject/cortex/pkg/chunk/testutils"
13-
"github.com/cortexproject/cortex/pkg/util"
1414
)
1515

1616
type fixture struct {
@@ -73,7 +73,7 @@ func dynamoDBFixture(provisionedErr, gangsize, maxParallelism int) testutils.Fix
7373
cfg: DynamoDBConfig{
7474
ChunkGangSize: gangsize,
7575
ChunkGetMaxParallelism: maxParallelism,
76-
BackoffConfig: util.BackoffConfig{
76+
BackoffConfig: backoff.Config{
7777
MinBackoff: 1 * time.Millisecond,
7878
MaxBackoff: 5 * time.Millisecond,
7979
MaxRetries: 20,

‎pkg/chunk/aws/retryer.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,22 @@ import (
55
"time"
66

77
"github.com/aws/aws-sdk-go/aws/request"
8+
"github.com/grafana/dskit/backoff"
89
ot "github.com/opentracing/opentracing-go"
910
otlog "github.com/opentracing/opentracing-go/log"
10-
11-
"github.com/cortexproject/cortex/pkg/util"
1211
)
1312

1413
// Map Cortex Backoff into AWS Retryer interface
1514
type retryer struct {
16-
*util.Backoff
15+
*backoff.Backoff
1716
maxRetries int
1817
}
1918

2019
var _ request.Retryer = &retryer{}
2120

22-
func newRetryer(ctx context.Context, cfg util.BackoffConfig) *retryer {
21+
func newRetryer(ctx context.Context, cfg backoff.Config) *retryer {
2322
return &retryer{
24-
Backoff: util.NewBackoff(ctx, cfg),
23+
Backoff: backoff.New(ctx, cfg),
2524
maxRetries: cfg.MaxRetries,
2625
}
2726
}

‎pkg/compactor/compactor.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515

1616
"github.com/go-kit/kit/log"
1717
"github.com/go-kit/kit/log/level"
18+
"github.com/grafana/dskit/backoff"
1819
"github.com/pkg/errors"
1920
"github.com/prometheus/client_golang/prometheus"
2021
"github.com/prometheus/client_golang/prometheus/promauto"
@@ -573,7 +574,7 @@ func (c *Compactor) compactUsers(ctx context.Context) {
573574
func (c *Compactor) compactUserWithRetries(ctx context.Context, userID string) error {
574575
var lastErr error
575576

576-
retries := util.NewBackoff(ctx, util.BackoffConfig{
577+
retries := backoff.New(ctx, backoff.Config{
577578
MinBackoff: c.compactorCfg.retryMinBackoff,
578579
MaxBackoff: c.compactorCfg.retryMaxBackoff,
579580
MaxRetries: c.compactorCfg.CompactionRetries,
@@ -670,7 +671,7 @@ func (c *Compactor) compactUser(ctx context.Context, userID string) error {
670671
func (c *Compactor) discoverUsersWithRetries(ctx context.Context) ([]string, error) {
671672
var lastErr error
672673

673-
retries := util.NewBackoff(ctx, util.BackoffConfig{
674+
retries := backoff.New(ctx, backoff.Config{
674675
MinBackoff: c.compactorCfg.retryMinBackoff,
675676
MaxBackoff: c.compactorCfg.retryMaxBackoff,
676677
MaxRetries: c.compactorCfg.CompactionRetries,

‎pkg/frontend/v2/frontend_scheduler_worker.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/go-kit/kit/log"
1010
"github.com/go-kit/kit/log/level"
11+
"github.com/grafana/dskit/backoff"
1112
"github.com/pkg/errors"
1213
"github.com/weaveworks/common/httpgrpc"
1314
"google.golang.org/grpc"
@@ -197,12 +198,12 @@ func (w *frontendSchedulerWorker) stop() {
197198
}
198199

199200
func (w *frontendSchedulerWorker) runOne(ctx context.Context, client schedulerpb.SchedulerForFrontendClient) {
200-
backoffConfig := util.BackoffConfig{
201+
backoffConfig := backoff.Config{
201202
MinBackoff: 50 * time.Millisecond,
202203
MaxBackoff: 1 * time.Second,
203204
}
204205

205-
backoff := util.NewBackoff(ctx, backoffConfig)
206+
backoff := backoff.New(ctx, backoffConfig)
206207
for backoff.Ongoing() {
207208
loop, loopErr := client.FrontendLoop(ctx)
208209
if loopErr != nil {

‎pkg/ingester/transfer.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"time"
1010

1111
"github.com/go-kit/kit/log/level"
12+
"github.com/grafana/dskit/backoff"
1213
"github.com/pkg/errors"
1314
"github.com/prometheus/common/model"
1415
"github.com/weaveworks/common/user"
@@ -17,7 +18,6 @@ import (
1718
"github.com/cortexproject/cortex/pkg/cortexpb"
1819
"github.com/cortexproject/cortex/pkg/ingester/client"
1920
"github.com/cortexproject/cortex/pkg/ring"
20-
"github.com/cortexproject/cortex/pkg/util"
2121
)
2222

2323
var (
@@ -275,7 +275,7 @@ func (i *Ingester) TransferOut(ctx context.Context) error {
275275
if i.cfg.MaxTransferRetries <= 0 {
276276
return ring.ErrTransferDisabled
277277
}
278-
backoff := util.NewBackoff(ctx, util.BackoffConfig{
278+
backoff := backoff.New(ctx, backoff.Config{
279279
MinBackoff: 100 * time.Millisecond,
280280
MaxBackoff: 5 * time.Second,
281281
MaxRetries: i.cfg.MaxTransferRetries,

‎pkg/querier/blocks_finder_bucket_scan.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111

1212
"github.com/go-kit/kit/log"
1313
"github.com/go-kit/kit/log/level"
14+
"github.com/grafana/dskit/backoff"
1415
"github.com/oklog/ulid"
1516
"github.com/pkg/errors"
1617
"github.com/prometheus/client_golang/prometheus"
@@ -270,7 +271,7 @@ pushJobsLoop:
270271
// scanUserBlocksWithRetries runs scanUserBlocks() retrying multiple times
271272
// in case of error.
272273
func (d *BucketScanBlocksFinder) scanUserBlocksWithRetries(ctx context.Context, userID string) (metas bucketindex.Blocks, deletionMarks map[ulid.ULID]*bucketindex.BlockDeletionMark, err error) {
273-
retries := util.NewBackoff(ctx, util.BackoffConfig{
274+
retries := backoff.New(ctx, backoff.Config{
274275
MinBackoff: time.Second,
275276
MaxBackoff: 30 * time.Second,
276277
MaxRetries: 3,

0 commit comments

Comments
 (0)