From 3744b7e365014230edaef6ff9d529cb2aa236ef5 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 6 Nov 2024 11:04:45 +0100 Subject: [PATCH 1/2] ktesting: abort entire test suite on SIGINT When aborting an integration test with CTRL-C while it runs, the current test fails and etcd exits. But additional tests were still being started and the failed slowly because they couldn't connect to etcd. It's better to fail additional tests in ktesting.Init when the test run has already been interrupted. While at it, also make it a bit more obvious that testing was interrupted by logging it and update one comment about this. Example: $ go test -v ./test/integration/quota ... I1106 11:42:48.857162 147325 etcd.go:416] "Not using watch cache" resource="events.events.k8s.io" I1106 11:42:48.857204 147325 handler.go:286] Adding GroupVersion events.k8s.io v1 to ResourceManager W1106 11:42:48.857209 147325 genericapiserver.go:765] Skipping API events.k8s.io/v1beta1 because it has no resources. ^C INFO: canceling context: received interrupt signal {"level":"warn","ts":"2024-11-06T11:42:48.984676+0100","caller":"embed/serve.go:160","msg":"stopping insecure grpc server due to error","error":"accept tcp 127.0.0.1:44177: use of closed network connection"} ... I1106 11:42:50.042430 147325 handler.go:142] kube-apiserver: GET "/apis/rbac.authorization.k8s.io/v1/clusterroles" satisfied by gorestful with webservice /apis/rbac.authorization.k8s.io/v1 test_server.go:241: timed out waiting for the condition --- FAIL: TestQuota (11.45s) === RUN TestQuotaLimitedResourceDenial quota_test.go:292: testing has been interrupted: received interrupt signal --- FAIL: TestQuotaLimitedResourceDenial (0.00s) === RUN TestQuotaLimitService quota_test.go:418: testing has been interrupted: received interrupt signal --- FAIL: TestQuotaLimitService (0.00s) FAIL --- test/utils/ktesting/ktesting.go | 1 + test/utils/ktesting/signals.go | 5 ++++- test/utils/ktesting/tcontext.go | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/test/utils/ktesting/ktesting.go b/test/utils/ktesting/ktesting.go index b48e3ea7014b6..774883f593261 100644 --- a/test/utils/ktesting/ktesting.go +++ b/test/utils/ktesting/ktesting.go @@ -43,6 +43,7 @@ func SetDefaultVerbosity(v int) { // NewTestContext is a replacement for ktesting.NewTestContext // which returns a more versatile context. func NewTestContext(tb testing.TB) (klog.Logger, TContext) { + tb.Helper() tCtx := Init(tb) return tCtx.Logger(), tCtx } diff --git a/test/utils/ktesting/signals.go b/test/utils/ktesting/signals.go index 1a9b9715fdc4b..0284d049d800b 100644 --- a/test/utils/ktesting/signals.go +++ b/test/utils/ktesting/signals.go @@ -19,6 +19,7 @@ package ktesting import ( "context" "errors" + "fmt" "io" "os" "os/signal" @@ -47,11 +48,13 @@ func init() { cancelCtx, cancel := context.WithCancelCause(context.Background()) go func() { <-signalCtx.Done() + fmt.Fprintf(os.Stdout, "\n\nINFO: canceling context: received interrupt signal\n\n") cancel(errors.New("received interrupt signal")) }() // This reimplements the contract between Ginkgo and Gomega for progress reporting. - // When using Ginkgo contexts, Ginkgo will implement it. This here is for "go test". + // When using Ginkgo contexts, Ginkgo will implement it. This here is for "go test" + // in combination with Gomega. // // nolint:staticcheck // It complains about using a plain string. This can only be fixed // by Ginkgo and Gomega formalizing this interface and define a type (somewhere... diff --git a/test/utils/ktesting/tcontext.go b/test/utils/ktesting/tcontext.go index 18bc387e057cb..49817e499e51d 100644 --- a/test/utils/ktesting/tcontext.go +++ b/test/utils/ktesting/tcontext.go @@ -239,7 +239,14 @@ func Init(tb TB, opts ...InitOption) TContext { Deadline() (time.Time, bool) }) + // If testing has been interrupted, immediately fail all new tests. + // Giving them a canceled context also works, but only if the tests + // actually check it. ctx := interruptCtx + if ctx.Err() != nil { + tb.Fatalf("testing has been interrupted: %v", context.Cause(ctx)) + } + if c.PerTestOutput { config := ktesting.NewConfig( ktesting.AnyToString(func(v interface{}) string { From 907297ed589699df5d35cd6de7527498f9f17269 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 6 Nov 2024 15:15:24 +0100 Subject: [PATCH 2/2] ktesting: print info and progress to /dev/tty The "received interrupt signal" is useful also when running with "go test" without -v because it shows that the shutdown has started. But more important is that a progress report gets shown because that feature is useful in particular when "go test" produces no output while it runs. --- test/utils/ktesting/signals.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test/utils/ktesting/signals.go b/test/utils/ktesting/signals.go index 0284d049d800b..45773f7eda8fe 100644 --- a/test/utils/ktesting/signals.go +++ b/test/utils/ktesting/signals.go @@ -41,6 +41,15 @@ type ginkgoReporter interface { } func init() { + // os.Stderr gets redirected by "go test". "go test -v" has to be + // used to see the output while a test runs. Opening /dev/tty avoids + // the redirection, but isn't always possible. + console, err := os.OpenFile("/dev/tty", os.O_RDWR|os.O_APPEND, 0) + if err != nil { + console = os.Stderr + fmt.Fprintf(console, "Cannot use /dev/tty: %v", err) + } + // Setting up signals is intentionally done in an init function because // then importing ktesting in a unit or integration test is sufficient // to activate the signal behavior. @@ -48,7 +57,7 @@ func init() { cancelCtx, cancel := context.WithCancelCause(context.Background()) go func() { <-signalCtx.Done() - fmt.Fprintf(os.Stdout, "\n\nINFO: canceling context: received interrupt signal\n\n") + fmt.Fprintf(console, "\n\nINFO: canceling test context(s): received interrupt signal\n\n") cancel(errors.New("received interrupt signal")) }() @@ -67,9 +76,7 @@ func init() { signal.Notify(defaultSignalChannel, progressSignals...) } - // os.Stderr gets redirected by "go test". "go test -v" has to be - // used to see the output while a test runs. - defaultProgressReporter.setOutput(os.Stderr) + defaultProgressReporter.setOutput(console) go defaultProgressReporter.run(interruptCtx, defaultSignalChannel) }