From 9a5e97d1fefb3e826707eb837d61b6cf52805f82 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Thu, 16 Dec 2021 09:54:24 -0600 Subject: [PATCH 01/12] V2 (#26) * Upgrade module version * Remove Listener Remove the Listener implementation in the interest of doing one thing and doing it well. We can add it back in another package down the line. * Update license year * Finish refactor * Remove ALL dependencies * Clean English * Improve minor code formatting * Another minor English improvement * Remove error nonsense We can always add it back later. * Clean english * Formatting? * Remove error vestiges --- LICENSE | 2 +- README.md | 51 ++++++++++++++++---- backoff.go | 54 --------------------- backoff_test.go | 58 ---------------------- doc.go | 2 +- go.mod | 9 +--- go.sum | 8 --- listener.go | 46 ------------------ listener_test.go | 110 ------------------------------------------ retrier.go | 36 ++++++++++++++ retrier_test.go | 17 +++++++ retry_example_test.go | 74 ---------------------------- 12 files changed, 98 insertions(+), 369 deletions(-) delete mode 100644 backoff.go delete mode 100644 backoff_test.go delete mode 100644 listener.go delete mode 100644 listener_test.go create mode 100644 retrier.go create mode 100644 retrier_test.go delete mode 100644 retry_example_test.go diff --git a/LICENSE b/LICENSE index 71d81d0..717af18 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2018 Coder Technologies Inc. +Copyright (c) 2021 Coder Technologies Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 2f03cc6..568af6b 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,49 @@ # retry -An expressive, flexible retry package for Go. +An exponentially backing off retry package for Go. -[![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](https://godoc.org/go.coder.com/retry) +[![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](https://godoc.org/github.com/coder/retry) -## Features +``` +go get github.com/coder/retry +``` -- Backoff helper -- Retrying net.Listener wrapper +## Features +- Offers a `for` loop experience instead of closures +- Only 2 exported methods +- No external dependencies ## Examples -See [retry_example_test.go](retry_example_test.go) - -## We're Hiring! - -If you're a passionate Go developer, send your resume and/or GitHub link to [jobs@coder.com](mailto:jobs@coder.com). +Wait for connectivity to google.com, checking at most once every +second. +```go +func pingGoogle(ctx context.Context) error { + var err error + r := retry.New(time.Second, time.Second*10) + for r.Wait(ctx) { + _, err = http.Get("https://google.com") + if err != nil { + continue + } + break + } + return err +} +``` + +Wait for connectivity to google.com, checking at most 10 times. +```go +func pingGoogle(ctx context.Context) error { + var err error + r := retry.New(time.Second, time.Second*10) + for n := 0; r.Wait(ctx) && n < 10; n++ { + _, err = http.Get("https://google.com") + if err != nil { + continue + } + break + } + return err +} +``` \ No newline at end of file diff --git a/backoff.go b/backoff.go deleted file mode 100644 index 518246c..0000000 --- a/backoff.go +++ /dev/null @@ -1,54 +0,0 @@ -package retry - -import ( - "context" - "time" - - "github.com/pkg/errors" -) - -// Backoff holds state about a backoff loop in which -// there should be a delay in iterations. -type Backoff struct { - // These two fields must be initialized. - // Floor should never be greater than or equal - // to the Ceil in general. If it is, the Backoff - // will stop backing off and just sleep for the Floor - // in Wait(). - Floor time.Duration - Ceil time.Duration - - delay time.Duration -} - -func (b *Backoff) backoff() { - if b.Floor >= b.Ceil { - return - } - - const growth = 2 - b.delay *= growth - if b.delay > b.Ceil { - b.delay = b.Ceil - } -} - -// Wait should be called at the end of the loop. It will sleep -// for the necessary duration before the next iteration of the loop -// can begin. -// If the context is cancelled, Wait will return early with a non-nil error. -func (b *Backoff) Wait(ctx context.Context) error { - if b.delay < b.Floor { - b.delay = b.Floor - } - - select { - case <-ctx.Done(): - return errors.Wrapf(ctx.Err(), "failed to sleep delay %v for retry attempt", b.delay) - case <-time.After(b.delay): - } - - b.backoff() - - return nil -} diff --git a/backoff_test.go b/backoff_test.go deleted file mode 100644 index da9c840..0000000 --- a/backoff_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package retry_test - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.coder.com/retry" -) - -func TestBackoff(t *testing.T) { - t.Parallel() - - t.Run("failure", func(t *testing.T) { - t.Parallel() - - start := time.Now() - - b := &retry.Backoff{ - Floor: time.Millisecond, - Ceil: time.Second * 5, - } - - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, time.Millisecond*100) - defer cancel() - - for time.Since(start) < time.Second { - err := b.Wait(ctx) - if err != nil { - return - } - } - - t.Errorf("succeeded: took: %v", time.Since(start)) - }) - - t.Run("success", func(t *testing.T) { - t.Parallel() - - start := time.Now() - - b := &retry.Backoff{ - Floor: time.Millisecond, - Ceil: time.Second * 5, - } - - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, time.Second*2) - defer cancel() - - for time.Since(start) < time.Second { - err := b.Wait(ctx) - require.NoError(t, err, "took: %v", time.Since(start)) - } - }) -} diff --git a/doc.go b/doc.go index 591ed4e..8d01b39 100644 --- a/doc.go +++ b/doc.go @@ -1,2 +1,2 @@ -// Package retry contains utilities for retrying an action until it succeeds. +// Package retry runs a failable function until it succeeds. package retry diff --git a/go.mod b/go.mod index b5ddf5f..12a1d62 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,3 @@ -module go.coder.com/retry +module github.com/coder/retry -require ( - github.com/davecgh/go-spew v1.1.0 // indirect - github.com/pkg/errors v0.8.0 - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/testify v1.1.4 -) +go 1.17 diff --git a/go.sum b/go.sum index 7fed76a..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +0,0 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.1.4 h1:ToftOQTytwshuOSj6bDSolVUa3GINfJP/fg3OkkOzQQ= -github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/listener.go b/listener.go deleted file mode 100644 index 2f2553f..0000000 --- a/listener.go +++ /dev/null @@ -1,46 +0,0 @@ -package retry - -import ( - "context" - "log" - "net" - "time" -) - -type Listener struct { - LogTmpErr func(err error) - net.Listener -} - -func (l Listener) Accept() (net.Conn, error) { - b := &Backoff{ - Floor: 5 * time.Millisecond, - Ceil: time.Second, - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - - for { - c, err := l.Listener.Accept() - if err == nil { - return c, nil - } - - ne, ok := err.(net.Error) - if !ok || !ne.Temporary() { - return nil, err - } - - if l.LogTmpErr == nil { - log.Printf("retry: temp error accepting next connection: %v", err) - } else { - l.LogTmpErr(err) - } - - err = b.Wait(ctx) - if err != nil { - return nil, err - } - } -} diff --git a/listener_test.go b/listener_test.go deleted file mode 100644 index 755f54d..0000000 --- a/listener_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package retry - -import ( - "net" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/require" -) - - -type testListener struct { - acceptFn func() (net.Conn, error) -} - -func newTestListener(acceptFn func() (net.Conn, error)) net.Listener { - return &Listener{ - LogTmpErr: func(err error) {}, - Listener: &testListener{ - acceptFn: acceptFn, - }, - } -} - -func (l *testListener) Accept() (net.Conn, error) { - return l.acceptFn() -} - -func (l *testListener) Close() error { - panic("stub") -} - -func (l *testListener) Addr() net.Addr { - panic("stub") -} - -type testNetError struct { - temporary bool -} - -func (e *testNetError) Error() string { - return "test net error" -} - -func (e *testNetError) Temporary() bool { - return e.temporary -} - -func (e *testNetError) Timeout() bool { - panic("do not call") -} - -func TestListener(t *testing.T) { - t.Parallel() - t.Run("general error", func(t *testing.T) { - t.Parallel() - - expectedErr := errors.New("general error") - acceptFn := func() (net.Conn, error) { - return nil, expectedErr - } - - _, err := newTestListener(acceptFn).Accept() - require.Equal(t, expectedErr, err) - }) - t.Run("success", func(t *testing.T) { - t.Parallel() - - acceptFn := func() (net.Conn, error) { - return nil, nil - } - - _, err := newTestListener(acceptFn).Accept() - require.Nil(t, err) - }) - t.Run("non temp net error", func(t *testing.T) { - t.Parallel() - - expectedErr := &testNetError{false} - acceptFn := func() (net.Conn, error) { - return nil, expectedErr - } - - _, err := newTestListener(acceptFn).Accept() - require.Equal(t, expectedErr, err) - }) - t.Run("3x temp net error", func(t *testing.T) { - t.Parallel() - - callCount := 0 - acceptFn := func() (net.Conn, error) { - callCount++ - switch callCount { - case 1: - return nil, &testNetError{true} - case 2: - return nil, &testNetError{true} - case 3: - return nil, nil - default: - t.Fatalf("test listener called too many times; callCount: %v", callCount) - panic("unreachable") - } - } - - _, err := newTestListener(acceptFn).Accept() - require.Nil(t, err) - require.Equal(t, callCount, 3) - }) -} diff --git a/retrier.go b/retrier.go new file mode 100644 index 0000000..cfcbe7d --- /dev/null +++ b/retrier.go @@ -0,0 +1,36 @@ +package retry + +import ( + "context" + "time" +) + +// Retrier implements an exponentially backing off retry instance. +// Use New instead of creating this object directly. +type Retrier struct { + delay time.Duration + floor, ceil time.Duration +} + +// New creates a retrier that exponentially backs off from floor to ceil pauses. +func New(floor, ceil time.Duration) *Retrier { + return &Retrier{ + delay: floor, + floor: floor, + ceil: ceil, + } +} + +func (r *Retrier) Wait(ctx context.Context) bool { + const growth = 2 + r.delay *= growth + if r.delay > r.ceil { + r.delay = r.ceil + } + select { + case <-time.After(r.delay): + return true + case <-ctx.Done(): + return false + } +} diff --git a/retrier_test.go b/retrier_test.go new file mode 100644 index 0000000..c6ac1e7 --- /dev/null +++ b/retrier_test.go @@ -0,0 +1,17 @@ +package retry + +import ( + "context" + "testing" + "time" +) + +func TestContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + r := New(time.Hour, time.Hour) + for r.Wait(ctx) { + t.Fatalf("attempt allowed even though context cancelled") + } +} diff --git a/retry_example_test.go b/retry_example_test.go deleted file mode 100644 index a334e4d..0000000 --- a/retry_example_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package retry_test - -import ( - "context" - "log" - "net" - "time" - - "go.coder.com/retry" -) - -func ExampleBackoffSuccess() { - start := time.Now() - - b := &retry.Backoff{ - Floor: time.Millisecond, - Ceil: time.Second * 5, - } - - ctx := context.Background() - - for time.Since(start) < time.Second { - err := b.Wait(ctx) - if err != nil { - log.Fatalf("failed: took: %v: err: %v", time.Since(start), err) - } - } - - log.Printf("success: took: %v", time.Since(start)) -} - -func ExampleBackoffError() { - start := time.Now() - - b := &retry.Backoff{ - Floor: time.Millisecond, - Ceil: time.Second * 5, - } - - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, time.Millisecond*100) - defer cancel() - - for time.Since(start) < time.Second { - err := b.Wait(ctx) - if err != nil { - log.Fatalf("failed: took: %v: err: %v", time.Since(start), err) - } - } - - log.Printf("success: took: %v", time.Since(start)) -} - -func ExampleListener() { - l, err := net.Listen("tcp", "localhost:0") - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - defer l.Close() - - l = retry.Listener{ - Listener: l, - } - - for { - c, err := l.Accept() - if err != nil { - log.Fatalf("failed to accept: %v", err) - } - defer c.Close() - - // ... - } -} From 5c2d84fb99965767513af9b1ef732bd99d833522 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Thu, 16 Dec 2021 09:56:18 -0600 Subject: [PATCH 02/12] Clean up README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 568af6b..dbeab6c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ go get github.com/coder/retry ``` ## Features -- Offers a `for` loop experience instead of closures +- A `for` loop experience instead of closures - Only 2 exported methods - No external dependencies From 33233f60ff266e7edfc61dfa2f49ed0772a066e1 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 18 Jan 2022 15:52:50 -0600 Subject: [PATCH 03/12] Simplify examples --- README.md | 34 +++++++++++++++++----------------- go.mod | 5 +++++ go.sum | 4 ++++ 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index dbeab6c..713154a 100644 --- a/README.md +++ b/README.md @@ -20,30 +20,30 @@ second. ```go func pingGoogle(ctx context.Context) error { var err error - r := retry.New(time.Second, time.Second*10) - for r.Wait(ctx) { - _, err = http.Get("https://google.com") - if err != nil { - continue - } - break - } - return err + + for r := retry.New(time.Second, time.Second*10); r.Wait(ctx); { + _, err = http.Get("https://google.com") + if err != nil { + continue + } + break + } + return err } ``` Wait for connectivity to google.com, checking at most 10 times. ```go func pingGoogle(ctx context.Context) error { - var err error - r := retry.New(time.Second, time.Second*10) - for n := 0; r.Wait(ctx) && n < 10; n++ { - _, err = http.Get("https://google.com") - if err != nil { - continue - } + var err error + + for r := retry.New(time.Second, time.Second*10); n := 0; r.Wait(ctx) && n < 10; n++ { + _, err = http.Get("https://google.com") + if err != nil { + continue + } break } - return err + return err } ``` \ No newline at end of file diff --git a/go.mod b/go.mod index 12a1d62..8e977a5 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module github.com/coder/retry go 1.17 + +require ( + github.com/ammario/cache v0.0.0-20211225003510-c969ec7f4766 // indirect + github.com/ammario/guard v0.0.0-20211225011814-e0e141dcbc37 // indirect +) diff --git a/go.sum b/go.sum index e69de29..81a60ac 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,4 @@ +github.com/ammario/cache v0.0.0-20211225003510-c969ec7f4766 h1:iptK1emgWyG+C67ygjlxqu5wLEPOQfy7xDx3M6qJbGs= +github.com/ammario/cache v0.0.0-20211225003510-c969ec7f4766/go.mod h1:qzIfV71nnVFkGq69iQ1OrwCDQTvqpvaUiu1Yz4xqpgI= +github.com/ammario/guard v0.0.0-20211225011814-e0e141dcbc37 h1:CnH2cF6w2k77PQcjcK9dp7x3kjaTM6rE7LSr+1frJQ0= +github.com/ammario/guard v0.0.0-20211225011814-e0e141dcbc37/go.mod h1:AADLzbLRXuBHZOn27mdHt67jRKZxbxf813//j0QsDds= From e90a2e1e091d8d0ef1370e728063bbad7c2e7d43 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Fri, 10 Feb 2023 17:54:34 +0200 Subject: [PATCH 04/12] fix: do first try immediately, then activate floor (#27) --- retrier.go | 5 ++++- retrier_test.go | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/retrier.go b/retrier.go index cfcbe7d..7ebfeab 100644 --- a/retrier.go +++ b/retrier.go @@ -15,7 +15,7 @@ type Retrier struct { // New creates a retrier that exponentially backs off from floor to ceil pauses. func New(floor, ceil time.Duration) *Retrier { return &Retrier{ - delay: floor, + delay: 0, floor: floor, ceil: ceil, } @@ -29,6 +29,9 @@ func (r *Retrier) Wait(ctx context.Context) bool { } select { case <-time.After(r.delay): + if r.delay < r.floor { + r.delay = r.floor + } return true case <-ctx.Done(): return false diff --git a/retrier_test.go b/retrier_test.go index c6ac1e7..0ac960e 100644 --- a/retrier_test.go +++ b/retrier_test.go @@ -15,3 +15,17 @@ func TestContextCancel(t *testing.T) { t.Fatalf("attempt allowed even though context cancelled") } } + +func TestFirstTryImmediately(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + r := New(time.Hour, time.Hour) + tt := time.Now() + if !r.Wait(ctx) { + t.Fatalf("attempt not allowed") + } + if time.Since(tt) > time.Second { + t.Fatalf("attempt took too long") + } +} From f27260cdd7995dbcb2881e60423b9aec3732f6ac Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Wed, 15 Feb 2023 21:19:31 -0600 Subject: [PATCH 05/12] Remove unnecessary dependencies I must've added these on accident. They were never used by this package. --- go.mod | 5 ----- go.sum | 4 ---- 2 files changed, 9 deletions(-) diff --git a/go.mod b/go.mod index 8e977a5..12a1d62 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,3 @@ module github.com/coder/retry go 1.17 - -require ( - github.com/ammario/cache v0.0.0-20211225003510-c969ec7f4766 // indirect - github.com/ammario/guard v0.0.0-20211225011814-e0e141dcbc37 // indirect -) diff --git a/go.sum b/go.sum index 81a60ac..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +0,0 @@ -github.com/ammario/cache v0.0.0-20211225003510-c969ec7f4766 h1:iptK1emgWyG+C67ygjlxqu5wLEPOQfy7xDx3M6qJbGs= -github.com/ammario/cache v0.0.0-20211225003510-c969ec7f4766/go.mod h1:qzIfV71nnVFkGq69iQ1OrwCDQTvqpvaUiu1Yz4xqpgI= -github.com/ammario/guard v0.0.0-20211225011814-e0e141dcbc37 h1:CnH2cF6w2k77PQcjcK9dp7x3kjaTM6rE7LSr+1frJQ0= -github.com/ammario/guard v0.0.0-20211225011814-e0e141dcbc37/go.mod h1:AADLzbLRXuBHZOn27mdHt67jRKZxbxf813//j0QsDds= From 76b40d712ea7af16e3b068fce1f100ab542081c1 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Thu, 16 Feb 2023 21:08:23 +0000 Subject: [PATCH 06/12] Replace MIT with CC0 --- LICENSE | 142 +++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 121 insertions(+), 21 deletions(-) diff --git a/LICENSE b/LICENSE index 717af18..cb9b058 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,121 @@ -The MIT License (MIT) - -Copyright (c) 2021 Coder Technologies Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be + protected by copyright and related or neighboring rights ("Copyright and + Related Rights"). Copyright and Related Rights include, but are not + limited to, the following: + +i. the right to reproduce, adapt, distribute, perform, display, +communicate, and translate a Work; +ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or +likeness depicted in a Work; +iv. rights protecting against unfair competition in regards to a Work, +subject to the limitations in paragraph 4(a), below; +v. rights protecting the extraction, dissemination, use and reuse of data +in a Work; +vi. database rights (such as those arising under Directive 96/9/EC of the +European Parliament and of the Council of 11 March 1996 on the legal +protection of databases, and under any national implementation +thereof, including any amended or successor version of such +directive); and +vii. other similar, equivalent or corresponding rights throughout the +world based on applicable law or treaty, and any national +implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention + of, applicable law, Affirmer hereby overtly, fully, permanently, + irrevocably and unconditionally waives, abandons, and surrenders all of + Affirmer's Copyright and Related Rights and associated claims and causes + of action, whether now known or unknown (including existing as well as + future claims and causes of action), in the Work (i) in all territories + worldwide, (ii) for the maximum duration provided by applicable law or + treaty (including future time extensions), (iii) in any current or future + medium and for any number of copies, and (iv) for any purpose whatsoever, + including without limitation commercial, advertising or promotional + purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each + member of the public at large and to the detriment of Affirmer's heirs and + successors, fully intending that such Waiver shall not be subject to + revocation, rescission, cancellation, termination, or any other legal or + equitable action to disrupt the quiet enjoyment of the Work by the public + as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason + be judged legally invalid or ineffective under applicable law, then the + Waiver shall be preserved to the maximum extent permitted taking into + account Affirmer's express Statement of Purpose. In addition, to the + extent the Waiver is so judged Affirmer hereby grants to each affected + person a royalty-free, non transferable, non sublicensable, non exclusive, + irrevocable and unconditional license to exercise Affirmer's Copyright and + Related Rights in the Work (i) in all territories worldwide, (ii) for the + maximum duration provided by applicable law or treaty (including future + time extensions), (iii) in any current or future medium and for any number + of copies, and (iv) for any purpose whatsoever, including without + limitation commercial, advertising or promotional purposes (the + "License"). The License shall be deemed effective as of the date CC0 was + applied by Affirmer to the Work. Should any part of the License for any + reason be judged legally invalid or ineffective under applicable law, such + partial invalidity or ineffectiveness shall not invalidate the remainder + of the License, and in such case Affirmer hereby affirms that he or she + will not (i) exercise any of his or her remaining Copyright and Related + Rights in the Work or (ii) assert any associated claims and causes of + action with respect to the Work, in either case contrary to Affirmer's + express Statement of Purpose. + +4. Limitations and Disclaimers. + +a. No trademark or patent rights held by Affirmer are waived, abandoned, +surrendered, licensed or otherwise affected by this document. +b. Affirmer offers the Work as-is and makes no representations or +warranties of any kind concerning the Work, express, implied, +statutory or otherwise, including without limitation warranties of +title, merchantability, fitness for a particular purpose, non +infringement, or the absence of latent or other defects, accuracy, or +the present or absence of errors, whether or not discoverable, all to +the greatest extent permissible under applicable law. +c. Affirmer disclaims responsibility for clearing rights of other persons +that may apply to the Work or any use thereof, including without +limitation any person's Copyright and Related Rights in the Work. +Further, Affirmer disclaims responsibility for obtaining any necessary +consents, permissions or other rights required for any use of the +Work. +d. Affirmer understands and acknowledges that Creative Commons is not a +party to this document and has no duty or obligation with respect to +this CC0 or use of the Work. From bdbabc9f0853a5f00583eff39303f7491f98d188 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Thu, 16 Feb 2023 21:08:39 +0000 Subject: [PATCH 07/12] General cleanup --- .travis.yml | 4 ---- doc.go | 2 +- go.mod | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fa90d4a..0000000 --- a/.travis.yml +++ /dev/null @@ -1,4 +0,0 @@ -language: go -go: -- "1.x" -go_import_path: go.coder.com/retry diff --git a/doc.go b/doc.go index 8d01b39..fcec039 100644 --- a/doc.go +++ b/doc.go @@ -1,2 +1,2 @@ -// Package retry runs a failable function until it succeeds. +// Package retry runs a fallible block of code until it succeeds. package retry diff --git a/go.mod b/go.mod index 12a1d62..075f312 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/coder/retry -go 1.17 +go 1.20 From 6c8802ca28e47d68364a7e0f4ed1ef7ffbe170c3 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Fri, 19 May 2023 22:11:38 -0500 Subject: [PATCH 08/12] Add Reset() method --- README.md | 2 +- retrier.go | 5 +++++ retrier_test.go | 9 +++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 713154a..ace68bb 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ go get github.com/coder/retry ## Features - A `for` loop experience instead of closures -- Only 2 exported methods +- Only 3 exported methods - No external dependencies ## Examples diff --git a/retrier.go b/retrier.go index 7ebfeab..007664c 100644 --- a/retrier.go +++ b/retrier.go @@ -37,3 +37,8 @@ func (r *Retrier) Wait(ctx context.Context) bool { return false } } + +// Reset resets the retrier to its initial state. +func (r *Retrier) Reset() { + r.delay = 0 +} diff --git a/retrier_test.go b/retrier_test.go index 0ac960e..f0949e1 100644 --- a/retrier_test.go +++ b/retrier_test.go @@ -29,3 +29,12 @@ func TestFirstTryImmediately(t *testing.T) { t.Fatalf("attempt took too long") } } + +func TestReset(t *testing.T) { + r := New(time.Hour, time.Hour) + // Should be immediate + ctx := context.Background() + r.Wait(ctx) + r.Reset() + r.Wait(ctx) +} From a8710231a1a7a7f884eb894aca0bee24c5caf21c Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Sun, 21 May 2023 12:39:12 -0500 Subject: [PATCH 09/12] Make minor format improvements to README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ace68bb..7864f43 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ An exponentially backing off retry package for Go. [![GoDoc](https://godoc.org/github.com/golang/gddo?status.svg)](https://godoc.org/github.com/coder/retry) ``` -go get github.com/coder/retry +go get github.com/coder/retry@latest ``` ## Features @@ -16,7 +16,7 @@ go get github.com/coder/retry ## Examples Wait for connectivity to google.com, checking at most once every -second. +second: ```go func pingGoogle(ctx context.Context) error { var err error @@ -32,7 +32,7 @@ func pingGoogle(ctx context.Context) error { } ``` -Wait for connectivity to google.com, checking at most 10 times. +Wait for connectivity to google.com, checking at most 10 times: ```go func pingGoogle(ctx context.Context) error { var err error From 12627b155ff59e5f62c15d262ba1ba06f17daa90 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Tue, 27 Jun 2023 12:47:18 -0500 Subject: [PATCH 10/12] Update README to give a goto example --- README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 7864f43..3a27635 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,29 @@ An exponentially backing off retry package for Go. go get github.com/coder/retry@latest ``` -## Features -- A `for` loop experience instead of closures -- Only 3 exported methods -- No external dependencies +`retry` promotes control flow using `for`/`goto` instead of callbacks, which are unwieldy in Go. ## Examples Wait for connectivity to google.com, checking at most once every second: + ```go func pingGoogle(ctx context.Context) error { var err error - - for r := retry.New(time.Second, time.Second*10); r.Wait(ctx); { - _, err = http.Get("https://google.com") - if err != nil { - continue + + r := retry.New(time.Second, time.Second*10); + + retry: + _, err = http.Get("https://google.com") + if err != nil { + if r.Wait(ctx) { + goto retry } - break + return err } - return err + + return nil } ``` From 14c7c27e14e40827a36754dd2071b09249d426f8 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Thu, 17 Aug 2023 12:22:32 -0500 Subject: [PATCH 11/12] Add support for Jitter (#28) * Expose Rate for better scaling control * Expose other Retry values to allow for direct instantiation * Use Phi as default scaling rate. --- README.md | 8 ++++++- retrier.go | 64 +++++++++++++++++++++++++++++++++++++++---------- retrier_test.go | 51 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3a27635..51799b5 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ An exponentially backing off retry package for Go. go get github.com/coder/retry@latest ``` -`retry` promotes control flow using `for`/`goto` instead of callbacks, which are unwieldy in Go. +`retry` promotes control flow using `for`/`goto` instead of callbacks. ## Examples @@ -21,6 +21,12 @@ func pingGoogle(ctx context.Context) error { r := retry.New(time.Second, time.Second*10); + // Jitter is useful when the majority of clients to a service use + // the same backoff policy. + // + // It is provided as a standard deviation. + r.Jitter = 0.1 + retry: _, err = http.Get("https://google.com") if err != nil { diff --git a/retrier.go b/retrier.go index 007664c..0cdcf30 100644 --- a/retrier.go +++ b/retrier.go @@ -2,35 +2,73 @@ package retry import ( "context" + "math" + "math/rand" "time" ) // Retrier implements an exponentially backing off retry instance. // Use New instead of creating this object directly. type Retrier struct { - delay time.Duration - floor, ceil time.Duration + // Delay is the current delay between attempts. + Delay time.Duration + + // Floor and Ceil are the minimum and maximum delays. + Floor, Ceil time.Duration + + // Rate is the rate at which the delay grows. + // E.g. 2 means the delay doubles each time. + Rate float64 + + // Jitter determines the level of indeterminism in the delay. + // + // It is the standard deviation of the normal distribution of a random variable + // multiplied by the delay. E.g. 0.1 means the delay is normally distributed + // with a standard deviation of 10% of the delay. Floor and Ceil are still + // respected, making outlandish values impossible. + // + // Jitter can help avoid thundering herds. + Jitter float64 } // New creates a retrier that exponentially backs off from floor to ceil pauses. func New(floor, ceil time.Duration) *Retrier { return &Retrier{ - delay: 0, - floor: floor, - ceil: ceil, + Delay: 0, + Floor: floor, + Ceil: ceil, + // Phi scales more calmly than 2, but still has nice pleasing + // properties. + Rate: math.Phi, + } +} + +func applyJitter(d time.Duration, jitter float64) time.Duration { + if jitter == 0 { + return d + } + d *= time.Duration(1 + jitter*rand.NormFloat64()) + if d < 0 { + return 0 } + return d } +// Wait returns after min(Delay*Growth, Ceil) or ctx is cancelled. +// The first call to Wait will return immediately. func (r *Retrier) Wait(ctx context.Context) bool { - const growth = 2 - r.delay *= growth - if r.delay > r.ceil { - r.delay = r.ceil + r.Delay *= time.Duration(float64(r.Delay) * r.Rate) + + r.Delay = applyJitter(r.Delay, r.Jitter) + + if r.Delay > r.Ceil { + r.Delay = r.Ceil } + select { - case <-time.After(r.delay): - if r.delay < r.floor { - r.delay = r.floor + case <-time.After(r.Delay): + if r.Delay < r.Floor { + r.Delay = r.Floor } return true case <-ctx.Done(): @@ -40,5 +78,5 @@ func (r *Retrier) Wait(ctx context.Context) bool { // Reset resets the retrier to its initial state. func (r *Retrier) Reset() { - r.delay = 0 + r.Delay = 0 } diff --git a/retrier_test.go b/retrier_test.go index f0949e1..10fcd4a 100644 --- a/retrier_test.go +++ b/retrier_test.go @@ -2,6 +2,7 @@ package retry import ( "context" + "math" "testing" "time" ) @@ -38,3 +39,53 @@ func TestReset(t *testing.T) { r.Reset() r.Wait(ctx) } + +func TestJitter_Normal(t *testing.T) { + t.Parallel() + + r := New(time.Millisecond, time.Millisecond) + r.Jitter = 0.5 + + var ( + sum time.Duration + waits []float64 + ctx = context.Background() + ) + for i := 0; i < 1000; i++ { + start := time.Now() + r.Wait(ctx) + took := time.Since(start) + waits = append(waits, (took.Seconds() * 1000)) + sum += took + } + + avg := float64(sum) / float64(len(waits)) + std := stdDev(waits) + if std > avg*0.1 { + t.Fatalf("standard deviation too high: %v", std) + } + + t.Logf("average: %v", time.Duration(avg)) + t.Logf("std dev: %v", std) + t.Logf("sample: %v", waits[len(waits)-10:]) +} + +// stdDev returns the standard deviation of the sample. +func stdDev(sample []float64) float64 { + if len(sample) == 0 { + return 0 + } + mean := 0.0 + for _, v := range sample { + mean += v + } + mean /= float64(len(sample)) + + variance := 0.0 + for _, v := range sample { + variance += math.Pow(v-mean, 2) + } + variance /= float64(len(sample)) + + return math.Sqrt(variance) +} From f5ccc4d2d45135bf65c7ccc5e78942dd7df19c84 Mon Sep 17 00:00:00 2001 From: Ammar Bandukwala Date: Sat, 11 Nov 2023 17:26:30 -0600 Subject: [PATCH 12/12] Fix double-scaling bug Turns out this package hasn't been working for a few months. --- example/main.go | 22 ++++++++++++++++++++++ retrier.go | 2 +- retrier_test.go | 21 +++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 example/main.go diff --git a/example/main.go b/example/main.go new file mode 100644 index 0000000..2f2abff --- /dev/null +++ b/example/main.go @@ -0,0 +1,22 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/coder/retry" +) + +func main() { + r := retry.New(time.Second, time.Second*10) + + ctx := context.Background() + + last := time.Now() + for r.Wait(ctx) { + // Do something that might fail + fmt.Printf("%v: hi\n", time.Since(last).Round(time.Second)) + last = time.Now() + } +} diff --git a/retrier.go b/retrier.go index 0cdcf30..a017341 100644 --- a/retrier.go +++ b/retrier.go @@ -57,7 +57,7 @@ func applyJitter(d time.Duration, jitter float64) time.Duration { // Wait returns after min(Delay*Growth, Ceil) or ctx is cancelled. // The first call to Wait will return immediately. func (r *Retrier) Wait(ctx context.Context) bool { - r.Delay *= time.Duration(float64(r.Delay) * r.Rate) + r.Delay = time.Duration(float64(r.Delay) * r.Rate) r.Delay = applyJitter(r.Delay, r.Jitter) diff --git a/retrier_test.go b/retrier_test.go index 10fcd4a..4eecf74 100644 --- a/retrier_test.go +++ b/retrier_test.go @@ -31,6 +31,27 @@ func TestFirstTryImmediately(t *testing.T) { } } +func TestScalesExponentially(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + r := New(time.Second, time.Second*10) + r.Rate = 2 + + start := time.Now() + + for i := 0; i < 3; i++ { + t.Logf("delay: %v", r.Delay) + r.Wait(ctx) + t.Logf("sinceStart: %v", time.Since(start).Round(time.Second)) + } + + sinceStart := time.Since(start).Round(time.Second) + if sinceStart != time.Second*6 { + t.Fatalf("did not scale correctly: %v", sinceStart) + } +} + func TestReset(t *testing.T) { r := New(time.Hour, time.Hour) // Should be immediate