Skip to content

chore: fix test flake in TestProvisionerd #9709

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 18, 2023
Merged

chore: fix test flake in TestProvisionerd #9709

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1b01d45
chore: fix test flake in TestProvisionerd
sreya Sep 15, 2023
356bca8
lint
sreya Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 37 additions & 19 deletions provisionerd/provisionerd_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func TestProvisionerd(t *testing.T) {
close(done)
})
closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{}), nil
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{}), nil
}, provisionerd.LocalProvisioners{})
require.NoError(t, closer.Close())
})
Expand Down Expand Up @@ -91,7 +91,7 @@ func TestProvisionerd(t *testing.T) {
completeChan := make(chan struct{})
closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
acquireJobAttempt := 0
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if acquireJobAttempt == 1 {
close(completeChan)
Expand All @@ -118,7 +118,7 @@ func TestProvisionerd(t *testing.T) {
var closerMutex sync.Mutex
closerMutex.Lock()
closer = createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
return &proto.AcquiredJob{
JobId: "test",
Expand Down Expand Up @@ -174,7 +174,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
return &proto.AcquiredJob{
JobId: "test",
Expand Down Expand Up @@ -214,7 +214,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
return &proto.AcquiredJob{
JobId: "test",
Expand Down Expand Up @@ -269,7 +269,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if !didAcquireJob.CAS(false, true) {
completeOnce.Do(func() { close(completeChan) })
Expand Down Expand Up @@ -361,7 +361,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if !didAcquireJob.CAS(false, true) {
completeOnce.Do(func() { close(completeChan) })
Expand Down Expand Up @@ -441,7 +441,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if !didAcquireJob.CAS(false, true) {
completeOnce.Do(func() { close(completeChan) })
Expand Down Expand Up @@ -513,7 +513,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if !didAcquireJob.CAS(false, true) {
completeOnce.Do(func() { close(completeChan) })
Expand Down Expand Up @@ -612,7 +612,7 @@ func TestProvisionerd(t *testing.T) {
)

closer := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if !didAcquireJob.CAS(false, true) {
completeOnce.Do(func() { close(completeChan) })
Expand Down Expand Up @@ -677,7 +677,7 @@ func TestProvisionerd(t *testing.T) {
updateChan := make(chan struct{})
completeChan := make(chan struct{})
server := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
return &proto.AcquiredJob{
JobId: "test",
Expand Down Expand Up @@ -755,7 +755,7 @@ func TestProvisionerd(t *testing.T) {
updateChan := make(chan struct{})
completeChan := make(chan struct{})
server := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
return &proto.AcquiredJob{
JobId: "test",
Expand Down Expand Up @@ -844,7 +844,7 @@ func TestProvisionerd(t *testing.T) {
completeOnce sync.Once
)
server := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
client := createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
client := createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if second.Load() {
return &proto.AcquiredJob{}, nil
Expand Down Expand Up @@ -927,7 +927,7 @@ func TestProvisionerd(t *testing.T) {
completeOnce sync.Once
)
server := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
client := createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
client := createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
if second.Load() {
completeOnce.Do(func() { close(completeChan) })
Expand Down Expand Up @@ -1006,7 +1006,7 @@ func TestProvisionerd(t *testing.T) {
completeOnce := sync.Once{}

server := createProvisionerd(t, func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
return createProvisionerDaemonClient(t, done, provisionerDaemonTestServer{
return createProvisionerDaemonClient(ctx, t, done, provisionerDaemonTestServer{
acquireJob: func(ctx context.Context, _ *proto.Empty) (*proto.AcquiredJob, error) {
m.Lock()
defer m.Unlock()
Expand Down Expand Up @@ -1118,7 +1118,7 @@ func createProvisionerd(t *testing.T, dialer provisionerd.Dialer, connector prov

// Creates a provisionerd protobuf client that's connected
// to the server implementation provided.
func createProvisionerDaemonClient(t *testing.T, done <-chan struct{}, server provisionerDaemonTestServer) proto.DRPCProvisionerDaemonClient {
func createProvisionerDaemonClient(ctx context.Context, t *testing.T, done <-chan struct{}, server provisionerDaemonTestServer) proto.DRPCProvisionerDaemonClient {
t.Helper()
if server.failJob == nil {
// Default to asserting the error from the failure, otherwise
Expand All @@ -1137,7 +1137,7 @@ func createProvisionerDaemonClient(t *testing.T, done <-chan struct{}, server pr
err := proto.DRPCRegisterProvisionerDaemon(mux, &server)
require.NoError(t, err)
srv := drpcserver.New(mux)
ctx, cancelFunc := context.WithCancel(context.Background())
ctx, cancelFunc := context.WithCancel(ctx)
closed := make(chan struct{})
go func() {
defer close(closed)
Expand All @@ -1148,13 +1148,31 @@ func createProvisionerDaemonClient(t *testing.T, done <-chan struct{}, server pr
<-closed
select {
case <-done:
t.Error("createProvisionerDaemonClient cleanup after test was done!")
// It's possible to get unlucky since the dialer is run in a retry in a goroutine.
// The following can occur:
// 1. The provisionerd.connect goroutine checks if we're closed prior to attempting to establish a connection
// with coderd, sees that we're not.
// 2. A test closes the server.
// 3. The provisionerd.connect goroutine calls the dialer to establish a connection. This
// function detects that someone has tried to create a client after the test finishes.
if ctx.Err() == nil {
t.Error("createProvisionerDaemonClient cleanup after test was done!")
}
default:
}
})
select {
case <-done:
t.Error("called createProvisionerDaemonClient after test was done!")
// It's possible to get unlucky since the dialer is run in a retry in a goroutine.
// The following can occur:
// 1. The provisionerd.connect goroutine checks if we're closed prior to attempting to establish a connection
// with coderd, sees that we're not.
// 2. A test closes the server.
// 3. The provisionerd.connect goroutine calls the dialer to establish a connection. This
// function detects that someone has tried to create a client after the test finishes.
if ctx.Err() == nil {
t.Error("createProvisionerDaemonClient cleanup after test was done!")
}
default:
}
return proto.NewDRPCProvisionerDaemonClient(clientPipe)
Expand Down