chore: skip timing-sensistive AgentMetadata test in the standard suite

ammario · ammario · commit 38340aa6ed7b · 2023-04-21T19:20:01.000Z
diff --git a/agent/agent_test.go b/agent/agent_test.go
@@ -1011,11 +1011,46 @@ func TestAgent_Metadata(t *testing.T) {
 	})
 
 	t.Run("Many", func(t *testing.T) {
+		t.Parallel()
+		script := "echo -n hello"
+		if runtime.GOOS == "windows" {
+			script = "powershell " + script
+		}
+		//nolint:dogsled
+		_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
+			Metadata: []codersdk.WorkspaceAgentMetadataDescription{
+				{
+					Key:      "greeting",
+					Interval: 1,
+					Script:   script,
+				},
+			},
+		}, 0)
+
+		var gotMd map[string]agentsdk.PostMetadataRequest
+		require.Eventually(t, func() bool {
+			gotMd = client.getMetadata()
+			return len(gotMd) == 1
+		}, testutil.WaitShort, testutil.IntervalMedium)
+
+		collectedAt1 := gotMd["greeting"].CollectedAt
+		assert.Equal(t, "hello", gotMd["greeting"].Value)
+
+		if !assert.Eventually(t, func() bool {
+			gotMd = client.getMetadata()
+			return gotMd["greeting"].CollectedAt.After(collectedAt1)
+		}, testutil.WaitShort, testutil.IntervalMedium) {
+			t.Fatalf("expected metadata to be collected again")
+		}
+	})
+
+	t.Run("ManyTiming", func(t *testing.T) {
 		if runtime.GOOS == "windows" {
 			// Shell scripting in Windows is a pain, and we have already tested
-			// that the OS logic works in the simpler "Once" test above.
+			// that the OS logic works in the simpler tests.
 			t.Skip()
 		}
+		testutil.SkipIfNotTiming(t)
 		t.Parallel()
 
 		dir := t.TempDir()
diff --git a/testutil/timing.go b/testutil/timing.go
@@ -0,0 +1,21 @@
+package testutil
+
+import (
+	"flag"
+	"testing"
+)
+
+// We can't run timing-sensitive tests in CI because of the
+// great variance in runner performance. Instead of not testing timing at all,
+// we relegate it to humans manually running certain tests with the "-timing"
+// flag from time to time.
+//
+// Eventually, we should run all timing tests in a self-hosted runner.
+
+var timingFlag = flag.Bool("timing", false, "run timing-sensitive tests")
+
+func SkipIfNotTiming(t *testing.T) {
+	if !*timingFlag {
+		t.Skip("skipping timing-sensitive test")
+	}
+}
