diff --git a/pkg/util/backoff.go b/pkg/util/backoff.go index b523b42238..3656f19cd0 100644 --- a/pkg/util/backoff.go +++ b/pkg/util/backoff.go @@ -24,25 +24,28 @@ func (cfg *BackoffConfig) RegisterFlags(prefix string, f *flag.FlagSet) { // Backoff implements exponential backoff with randomized wait times type Backoff struct { - cfg BackoffConfig - ctx context.Context - numRetries int - duration time.Duration + cfg BackoffConfig + ctx context.Context + numRetries int + nextDelayMin time.Duration + nextDelayMax time.Duration } // NewBackoff creates a Backoff object. Pass a Context that can also terminate the operation. func NewBackoff(ctx context.Context, cfg BackoffConfig) *Backoff { return &Backoff{ - cfg: cfg, - ctx: ctx, - duration: cfg.MinBackoff, + cfg: cfg, + ctx: ctx, + nextDelayMin: cfg.MinBackoff, + nextDelayMax: doubleDuration(cfg.MinBackoff, cfg.MaxBackoff), } } // Reset the Backoff back to its initial condition func (b *Backoff) Reset() { b.numRetries = 0 - b.duration = b.cfg.MinBackoff + b.nextDelayMin = b.cfg.MinBackoff + b.nextDelayMax = doubleDuration(b.cfg.MinBackoff, b.cfg.MaxBackoff) } // Ongoing returns true if caller should keep going @@ -70,18 +73,45 @@ func (b *Backoff) NumRetries() int { // Wait sleeps for the backoff time then increases the retry count and backoff time // Returns immediately if Context is terminated func (b *Backoff) Wait() { - b.numRetries++ - // Based on the "Full Jitter" approach from https://www.awsarchitectureblog.com/2015/03/backoff.html - // sleep = random_between(0, min(cap, base * 2 ** attempt)) + // Increase the number of retries and get the next delay + sleepTime := b.nextDelay() + if b.Ongoing() { - sleepTime := time.Duration(rand.Int63n(int64(b.duration))) select { case <-b.ctx.Done(): case <-time.After(sleepTime): } } - b.duration = b.duration * 2 - if b.duration > b.cfg.MaxBackoff { - b.duration = b.cfg.MaxBackoff +} + +func (b *Backoff) nextDelay() time.Duration { + b.numRetries++ + + // Handle the edge case the min and max have the same value + // (or due to some misconfig max is < min) + if b.nextDelayMin >= b.nextDelayMax { + return b.nextDelayMin } + + // Add a jitter within the next exponential backoff range + sleepTime := b.nextDelayMin + time.Duration(rand.Int63n(int64(b.nextDelayMax-b.nextDelayMin))) + + // Apply the exponential backoff to calculate the next jitter + // range, unless we've already reached the max + if b.nextDelayMax < b.cfg.MaxBackoff { + b.nextDelayMin = doubleDuration(b.nextDelayMin, b.cfg.MaxBackoff) + b.nextDelayMax = doubleDuration(b.nextDelayMax, b.cfg.MaxBackoff) + } + + return sleepTime +} + +func doubleDuration(value time.Duration, max time.Duration) time.Duration { + value = value * 2 + + if value <= max { + return value + } + + return max } diff --git a/pkg/util/backoff_test.go b/pkg/util/backoff_test.go new file mode 100644 index 0000000000..2202ad5264 --- /dev/null +++ b/pkg/util/backoff_test.go @@ -0,0 +1,103 @@ +package util + +import ( + "context" + "testing" + "time" +) + +func TestBackoff_nextDelay(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + minBackoff time.Duration + maxBackoff time.Duration + expectedRanges [][]time.Duration + }{ + "exponential backoff with jitter honoring min and max": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 10 * time.Second, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + {800 * time.Millisecond, 1600 * time.Millisecond}, + {1600 * time.Millisecond, 3200 * time.Millisecond}, + {3200 * time.Millisecond, 6400 * time.Millisecond}, + {6400 * time.Millisecond, 10000 * time.Millisecond}, + {6400 * time.Millisecond, 10000 * time.Millisecond}, + }, + }, + "exponential backoff with max equal to the end of a range": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 800 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + }, + }, + "exponential backoff with max equal to the end of a range + 1": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 801 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 800 * time.Millisecond}, + {800 * time.Millisecond, 801 * time.Millisecond}, + {800 * time.Millisecond, 801 * time.Millisecond}, + }, + }, + "exponential backoff with max equal to the end of a range - 1": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 799 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 400 * time.Millisecond}, + {400 * time.Millisecond, 799 * time.Millisecond}, + {400 * time.Millisecond, 799 * time.Millisecond}, + }, + }, + "min backoff is equal to max": { + minBackoff: 100 * time.Millisecond, + maxBackoff: 100 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {100 * time.Millisecond, 100 * time.Millisecond}, + {100 * time.Millisecond, 100 * time.Millisecond}, + {100 * time.Millisecond, 100 * time.Millisecond}, + }, + }, + "min backoff is greater then max": { + minBackoff: 200 * time.Millisecond, + maxBackoff: 100 * time.Millisecond, + expectedRanges: [][]time.Duration{ + {200 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 200 * time.Millisecond}, + {200 * time.Millisecond, 200 * time.Millisecond}, + }, + }, + } + + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + t.Parallel() + + b := NewBackoff(context.Background(), BackoffConfig{ + MinBackoff: testData.minBackoff, + MaxBackoff: testData.maxBackoff, + MaxRetries: len(testData.expectedRanges), + }) + + for _, expectedRange := range testData.expectedRanges { + delay := b.nextDelay() + + if delay < expectedRange[0] || delay > expectedRange[1] { + t.Errorf("%d expected to be within %d and %d", delay, expectedRange[0], expectedRange[1]) + } + } + }) + } +}