From a8710231a1a7a7f884eb894aca0bee24c5caf21c Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Sun, 21 May 2023 12:39:12 -0500 Subject: [PATCH 1/4] Make minor format improvements to README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ace68bb..7864f43 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ An exponentially backing off retry package for Go. [![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](https://godoc.org/github.com/coder/retry) ``` -go get github.com/coder/retry +go get github.com/coder/retry@latest ``` ## Features @@ -16,7 +16,7 @@ go get github.com/coder/retry ## Examples Wait for connectivity to google.com, checking at most once every -second. +second: ```go func pingGoogle(ctx context.Context) error { var err error @@ -32,7 +32,7 @@ func pingGoogle(ctx context.Context) error { } ``` -Wait for connectivity to google.com, checking at most 10 times. +Wait for connectivity to google.com, checking at most 10 times: ```go func pingGoogle(ctx context.Context) error { var err error From 12627b155ff59e5f62c15d262ba1ba06f17daa90 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 27 Jun 2023 12:47:18 -0500 Subject: [PATCH 2/4] Update README to give a goto example --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7864f43..3a27635 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,29 @@ An exponentially backing off retry package for Go. go get github.com/coder/retry@latest ``` -## Features -- A `for` loop experience instead of closures -- Only 3 exported methods -- No external dependencies +`retry` promotes control flow using `for`/`goto` instead of callbacks, which are unwieldy in Go. ## Examples Wait for connectivity to google.com, checking at most once every second: + ```go func pingGoogle(ctx context.Context) error { var err error - - for r := retry.New(time.Second, time.Second*10); r.Wait(ctx); { - _, err = http.Get("https://google.com") - if err != nil { - continue + + r := retry.New(time.Second, time.Second*10); + + retry: + _, err = http.Get("https://google.com") + if err != nil { + if r.Wait(ctx) { + goto retry } - break + return err } - return err + + return nil } ``` From 14c7c27e14e40827a36754dd2071b09249d426f8 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Thu, 17 Aug 2023 12:22:32 -0500 Subject: [PATCH 3/4] Add support for Jitter (#28) * Expose Rate for better scaling control * Expose other Retry values to allow for direct instantiation * Use Phi as default scaling rate. --- README.md | 8 ++++++- retrier.go | 64 +++++++++++++++++++++++++++++++++++++++---------- retrier_test.go | 51 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3a27635..51799b5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ An exponentially backing off retry package for Go. go get github.com/coder/retry@latest ``` -`retry` promotes control flow using `for`/`goto` instead of callbacks, which are unwieldy in Go. +`retry` promotes control flow using `for`/`goto` instead of callbacks. ## Examples @@ -21,6 +21,12 @@ func pingGoogle(ctx context.Context) error { r := retry.New(time.Second, time.Second*10); + // Jitter is useful when the majority of clients to a service use + // the same backoff policy. + // + // It is provided as a standard deviation. + r.Jitter = 0.1 + retry: _, err = http.Get("https://google.com") if err != nil { diff --git a/retrier.go b/retrier.go index 007664c..0cdcf30 100644 --- a/retrier.go +++ b/retrier.go @@ -2,35 +2,73 @@ package retry import ( "context" + "math" + "math/rand" "time" ) // Retrier implements an exponentially backing off retry instance. // Use New instead of creating this object directly. type Retrier struct { - delay time.Duration - floor, ceil time.Duration + // Delay is the current delay between attempts. + Delay time.Duration + + // Floor and Ceil are the minimum and maximum delays. + Floor, Ceil time.Duration + + // Rate is the rate at which the delay grows. + // E.g. 2 means the delay doubles each time. + Rate float64 + + // Jitter determines the level of indeterminism in the delay. + // + // It is the standard deviation of the normal distribution of a random variable + // multiplied by the delay. E.g. 0.1 means the delay is normally distributed + // with a standard deviation of 10% of the delay. Floor and Ceil are still + // respected, making outlandish values impossible. + // + // Jitter can help avoid thundering herds. + Jitter float64 } // New creates a retrier that exponentially backs off from floor to ceil pauses. func New(floor, ceil time.Duration) *Retrier { return &Retrier{ - delay: 0, - floor: floor, - ceil: ceil, + Delay: 0, + Floor: floor, + Ceil: ceil, + // Phi scales more calmly than 2, but still has nice pleasing + // properties. + Rate: math.Phi, + } +} + +func applyJitter(d time.Duration, jitter float64) time.Duration { + if jitter == 0 { + return d + } + d *= time.Duration(1 + jitter*rand.NormFloat64()) + if d < 0 { + return 0 } + return d } +// Wait returns after min(Delay*Growth, Ceil) or ctx is cancelled. +// The first call to Wait will return immediately. func (r *Retrier) Wait(ctx context.Context) bool { - const growth = 2 - r.delay *= growth - if r.delay > r.ceil { - r.delay = r.ceil + r.Delay *= time.Duration(float64(r.Delay) * r.Rate) + + r.Delay = applyJitter(r.Delay, r.Jitter) + + if r.Delay > r.Ceil { + r.Delay = r.Ceil } + select { - case <-time.After(r.delay): - if r.delay < r.floor { - r.delay = r.floor + case <-time.After(r.Delay): + if r.Delay < r.Floor { + r.Delay = r.Floor } return true case <-ctx.Done(): @@ -40,5 +78,5 @@ func (r *Retrier) Wait(ctx context.Context) bool { // Reset resets the retrier to its initial state. func (r *Retrier) Reset() { - r.delay = 0 + r.Delay = 0 } diff --git a/retrier_test.go b/retrier_test.go index f0949e1..10fcd4a 100644 --- a/retrier_test.go +++ b/retrier_test.go @@ -2,6 +2,7 @@ package retry import ( "context" + "math" "testing" "time" ) @@ -38,3 +39,53 @@ func TestReset(t *testing.T) { r.Reset() r.Wait(ctx) } + +func TestJitter_Normal(t *testing.T) { + t.Parallel() + + r := New(time.Millisecond, time.Millisecond) + r.Jitter = 0.5 + + var ( + sum time.Duration + waits []float64 + ctx = context.Background() + ) + for i := 0; i < 1000; i++ { + start := time.Now() + r.Wait(ctx) + took := time.Since(start) + waits = append(waits, (took.Seconds() * 1000)) + sum += took + } + + avg := float64(sum) / float64(len(waits)) + std := stdDev(waits) + if std > avg*0.1 { + t.Fatalf("standard deviation too high: %v", std) + } + + t.Logf("average: %v", time.Duration(avg)) + t.Logf("std dev: %v", std) + t.Logf("sample: %v", waits[len(waits)-10:]) +} + +// stdDev returns the standard deviation of the sample. +func stdDev(sample []float64) float64 { + if len(sample) == 0 { + return 0 + } + mean := 0.0 + for _, v := range sample { + mean += v + } + mean /= float64(len(sample)) + + variance := 0.0 + for _, v := range sample { + variance += math.Pow(v-mean, 2) + } + variance /= float64(len(sample)) + + return math.Sqrt(variance) +} From f5ccc4d2d45135bf65c7ccc5e78942dd7df19c84 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Sat, 11 Nov 2023 17:26:30 -0600 Subject: [PATCH 4/4] Fix double-scaling bug Turns out this package hasn't been working for a few months. --- example/main.go | 22 ++++++++++++++++++++++ retrier.go | 2 +- retrier_test.go | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 example/main.go diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..2f2abff --- /dev/null +++ b/example/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/coder/retry" +) + +func main() { + r := retry.New(time.Second, time.Second*10) + + ctx := context.Background() + + last := time.Now() + for r.Wait(ctx) { + // Do something that might fail + fmt.Printf("%v: hi\n", time.Since(last).Round(time.Second)) + last = time.Now() + } +} diff --git a/retrier.go b/retrier.go index 0cdcf30..a017341 100644 --- a/retrier.go +++ b/retrier.go @@ -57,7 +57,7 @@ func applyJitter(d time.Duration, jitter float64) time.Duration { // Wait returns after min(Delay*Growth, Ceil) or ctx is cancelled. // The first call to Wait will return immediately. func (r *Retrier) Wait(ctx context.Context) bool { - r.Delay *= time.Duration(float64(r.Delay) * r.Rate) + r.Delay = time.Duration(float64(r.Delay) * r.Rate) r.Delay = applyJitter(r.Delay, r.Jitter) diff --git a/retrier_test.go b/retrier_test.go index 10fcd4a..4eecf74 100644 --- a/retrier_test.go +++ b/retrier_test.go @@ -31,6 +31,27 @@ func TestFirstTryImmediately(t *testing.T) { } } +func TestScalesExponentially(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + r := New(time.Second, time.Second*10) + r.Rate = 2 + + start := time.Now() + + for i := 0; i < 3; i++ { + t.Logf("delay: %v", r.Delay) + r.Wait(ctx) + t.Logf("sinceStart: %v", time.Since(start).Round(time.Second)) + } + + sinceStart := time.Since(start).Round(time.Second) + if sinceStart != time.Second*6 { + t.Fatalf("did not scale correctly: %v", sinceStart) + } +} + func TestReset(t *testing.T) { r := New(time.Hour, time.Hour) // Should be immediate