Switched backoff implementation from a 'Full Jitter' to a 'Ranged Jitter'

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

pkg/util/backoff.go

@@ -24,25 +24,28 @@ func (cfg *BackoffConfig) RegisterFlags(prefix string, f *flag.FlagSet) {
 
 // Backoff implements exponential backoff with randomized wait times
 type Backoff struct {
-	cfg        BackoffConfig
-	ctx        context.Context
-	numRetries int
-	duration   time.Duration
+	cfg          BackoffConfig
+	ctx          context.Context
+	numRetries   int
+	nextDelayMin time.Duration
+	nextDelayMax time.Duration
 }
 
 // NewBackoff creates a Backoff object. Pass a Context that can also terminate the operation.
 func NewBackoff(ctx context.Context, cfg BackoffConfig) *Backoff {
 	return &Backoff{
-		cfg:      cfg,
-		ctx:      ctx,
-		duration: cfg.MinBackoff,
+		cfg:          cfg,
+		ctx:          ctx,
+		nextDelayMin: cfg.MinBackoff,
+		nextDelayMax: doubleDuration(cfg.MinBackoff, cfg.MaxBackoff),
 	}
 }
 
 // Reset the Backoff back to its initial condition
 func (b *Backoff) Reset() {
 	b.numRetries = 0
-	b.duration = b.cfg.MinBackoff
+	b.nextDelayMin = b.cfg.MinBackoff
+	b.nextDelayMax = doubleDuration(b.cfg.MinBackoff, b.cfg.MaxBackoff)
 }
 
 // Ongoing returns true if caller should keep going
@@ -70,18 +73,45 @@ func (b *Backoff) NumRetries() int {
 // Wait sleeps for the backoff time then increases the retry count and backoff time
 // Returns immediately if Context is terminated
 func (b *Backoff) Wait() {
-	b.numRetries++
-	// Based on the "Full Jitter" approach from https://www.awsarchitectureblog.com/2015/03/backoff.html
-	// sleep = random_between(0, min(cap, base * 2 ** attempt))
+	// Increase the number of retries and get the next delay
+	sleepTime := b.nextDelay()
+
 	if b.Ongoing() {
-		sleepTime := time.Duration(rand.Int63n(int64(b.duration)))
 		select {
 		case <-b.ctx.Done():
 		case <-time.After(sleepTime):
 		}
 	}
-	b.duration = b.duration * 2
-	if b.duration > b.cfg.MaxBackoff {
-		b.duration = b.cfg.MaxBackoff
+}
+
+func (b *Backoff) nextDelay() time.Duration {
+	b.numRetries++
+
+	// Handle the edge case the min and max have the same value
+	// (or due to some misconfig max is < min)
+	if b.nextDelayMin >= b.nextDelayMax {
+		return b.nextDelayMin
 	}
+
+	// Add a jitter within the next exponential backoff range
+	sleepTime := b.nextDelayMin + time.Duration(rand.Int63n(int64(b.nextDelayMax-b.nextDelayMin)))
+
+	// Apply the exponential backoff to calculate the next jitter
+	// range, unless we've already reached the max
+	if b.nextDelayMax < b.cfg.MaxBackoff {
+		b.nextDelayMin = doubleDuration(b.nextDelayMin, b.cfg.MaxBackoff)
+		b.nextDelayMax = doubleDuration(b.nextDelayMax, b.cfg.MaxBackoff)
+	}
+
+	return sleepTime
+}
+
+func doubleDuration(value time.Duration, max time.Duration) time.Duration {
+	value = value * 2
+
+	if value <= max {
+		return value
+	}
+
+	return max
 }

pkg/util/backoff_test.go

@@ -0,0 +1,103 @@
+package util
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestBackoff_nextDelay(t *testing.T) {
+	t.Parallel()
+
+	tests := map[string]struct {
+		minBackoff     time.Duration
+		maxBackoff     time.Duration
+		expectedRanges [][]time.Duration
+	}{
+		"exponential backoff with jitter honoring min and max": {
+			minBackoff: 100 * time.Millisecond,
+			maxBackoff: 10 * time.Second,
+			expectedRanges: [][]time.Duration{
+				{100 * time.Millisecond, 200 * time.Millisecond},
+				{200 * time.Millisecond, 400 * time.Millisecond},
+				{400 * time.Millisecond, 800 * time.Millisecond},
+				{800 * time.Millisecond, 1600 * time.Millisecond},
+				{1600 * time.Millisecond, 3200 * time.Millisecond},
+				{3200 * time.Millisecond, 6400 * time.Millisecond},
+				{6400 * time.Millisecond, 10000 * time.Millisecond},
+				{6400 * time.Millisecond, 10000 * time.Millisecond},
+			},
+		},
+		"exponential backoff with max equal to the end of a range": {
+			minBackoff: 100 * time.Millisecond,
+			maxBackoff: 800 * time.Millisecond,
+			expectedRanges: [][]time.Duration{
+				{100 * time.Millisecond, 200 * time.Millisecond},
+				{200 * time.Millisecond, 400 * time.Millisecond},
+				{400 * time.Millisecond, 800 * time.Millisecond},
+				{400 * time.Millisecond, 800 * time.Millisecond},
+			},
+		},
+		"exponential backoff with max equal to the end of a range + 1": {
+			minBackoff: 100 * time.Millisecond,
+			maxBackoff: 801 * time.Millisecond,
+			expectedRanges: [][]time.Duration{
+				{100 * time.Millisecond, 200 * time.Millisecond},
+				{200 * time.Millisecond, 400 * time.Millisecond},
+				{400 * time.Millisecond, 800 * time.Millisecond},
+				{800 * time.Millisecond, 801 * time.Millisecond},
+				{800 * time.Millisecond, 801 * time.Millisecond},
+			},
+		},
+		"exponential backoff with max equal to the end of a range - 1": {
+			minBackoff: 100 * time.Millisecond,
+			maxBackoff: 799 * time.Millisecond,
+			expectedRanges: [][]time.Duration{
+				{100 * time.Millisecond, 200 * time.Millisecond},
+				{200 * time.Millisecond, 400 * time.Millisecond},
+				{400 * time.Millisecond, 799 * time.Millisecond},
+				{400 * time.Millisecond, 799 * time.Millisecond},
+			},
+		},
+		"min backoff is equal to max": {
+			minBackoff: 100 * time.Millisecond,
+			maxBackoff: 100 * time.Millisecond,
+			expectedRanges: [][]time.Duration{
+				{100 * time.Millisecond, 100 * time.Millisecond},
+				{100 * time.Millisecond, 100 * time.Millisecond},
+				{100 * time.Millisecond, 100 * time.Millisecond},
+			},
+		},
+		"min backoff is greater then max": {
+			minBackoff: 200 * time.Millisecond,
+			maxBackoff: 100 * time.Millisecond,
+			expectedRanges: [][]time.Duration{
+				{200 * time.Millisecond, 200 * time.Millisecond},
+				{200 * time.Millisecond, 200 * time.Millisecond},
+				{200 * time.Millisecond, 200 * time.Millisecond},
+			},
+		},
+	}
+
+	for testName, testData := range tests {
+		testData := testData
+
+		t.Run(testName, func(t *testing.T) {
+			t.Parallel()
+
+			b := NewBackoff(context.Background(), BackoffConfig{
+				MinBackoff: testData.minBackoff,
+				MaxBackoff: testData.maxBackoff,
+				MaxRetries: len(testData.expectedRanges),
+			})
+
+			for _, expectedRange := range testData.expectedRanges {
+				delay := b.nextDelay()
+
+				if delay < expectedRange[0] || delay > expectedRange[1] {
+					t.Errorf("%d expected to be within %d and %d", delay, expectedRange[0], expectedRange[1])
+				}
+			}
+		})
+	}
+}