Skip to content

Commit ca0125e

Browse files
ammarioammario
committed
chore: skip timing-sensistive AgentMetadata test in the standard suite
1 parent 501dfee commit ca0125e

File tree

2 files changed

+109
-53
lines changed

2 files changed

+109
-53
lines changed

‎agent/agent_test.go

Lines changed: 88 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1011,78 +1011,113 @@ func TestAgent_Metadata(t *testing.T) {
10111011
})
10121012

10131013
t.Run("Many", func(t *testing.T) {
1014+
t.Parallel()
1015+
script := "echo -n hello"
10141016
if runtime.GOOS == "windows" {
1015-
// Shell scripting in Windows is a pain, and we have already tested
1016-
// that the OS logic works in the simpler "Once" test above.
1017-
t.Skip()
1017+
script = "powershell " + script
10181018
}
1019-
t.Parallel()
1020-
1021-
dir := t.TempDir()
1022-
1023-
const reportInterval = 2
1024-
const intervalUnit = 100 * time.Millisecond
1025-
var (
1026-
greetingPath = filepath.Join(dir, "greeting")
1027-
script = "echo hello | tee -a " + greetingPath
1028-
)
1019+
//nolint:dogsled
10291020
_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
10301021
Metadata: []codersdk.WorkspaceAgentMetadataDescription{
10311022
{
10321023
Key: "greeting",
1033-
Interval: reportInterval,
1024+
Interval: 1,
10341025
Script: script,
10351026
},
1036-
{
1037-
Key: "bad",
1038-
Interval: reportInterval,
1039-
Script: "exit 1",
1040-
},
10411027
},
10421028
}, 0)
10431029

1030+
var gotMd map[string]agentsdk.PostMetadataRequest
10441031
require.Eventually(t, func() bool {
1045-
return len(client.getMetadata()) == 2
1032+
gotMd = client.getMetadata()
1033+
return len(gotMd) == 1
10461034
}, testutil.WaitShort, testutil.IntervalMedium)
10471035

1048-
for start := time.Now(); time.Since(start) < testutil.WaitMedium; time.Sleep(testutil.IntervalMedium) {
1049-
md := client.getMetadata()
1050-
if len(md) != 2 {
1051-
panic("unexpected number of metadata entries")
1052-
}
1036+
collectedAt1 := gotMd["greeting"].CollectedAt
1037+
assert.Equal(t, "hello", gotMd["greeting"].Value)
10531038

1054-
require.Equal(t, "hello\n", md["greeting"].Value)
1055-
require.Equal(t, "exit status 1", md["bad"].Error)
1039+
if !assert.Eventually(t, func() bool {
1040+
gotMd = client.getMetadata()
1041+
return gotMd["greeting"].CollectedAt.After(collectedAt1)
1042+
}, testutil.WaitShort, testutil.IntervalMedium) {
1043+
t.Fatalf("expected metadata to be collected again")
1044+
}
1045+
})
1046+
}
10561047

1057-
greetingByt, err := os.ReadFile(greetingPath)
1058-
require.NoError(t, err)
1048+
func TestAgentMetadata_Timing(t *testing.T) {
1049+
if runtime.GOOS == "windows" {
1050+
// Shell scripting in Windows is a pain, and we have already tested
1051+
// that the OS logic works in the simpler tests.
1052+
t.Skip()
1053+
}
1054+
testutil.SkipIfNotTiming(t)
1055+
t.Parallel()
10591056

1060-
var (
1061-
numGreetings = bytes.Count(greetingByt, []byte("hello"))
1062-
idealNumGreetings = time.Since(start) / (reportInterval * intervalUnit)
1063-
// We allow a 50% error margin because the report loop may backlog
1064-
// in CI and other toasters. In production, there is no hard
1065-
// guarantee on timing either, and the frontend gives similar
1066-
// wiggle room to the staleness of the value.
1067-
upperBound = int(idealNumGreetings) + 1
1068-
lowerBound = (int(idealNumGreetings) / 2)
1069-
)
1070-
1071-
if idealNumGreetings < 50 {
1072-
// There is an insufficient sample size.
1073-
continue
1074-
}
1057+
dir := t.TempDir()
10751058

1076-
t.Logf("numGreetings: %d, idealNumGreetings: %d", numGreetings, idealNumGreetings)
1077-
// The report loop may slow down on load, but it should never, ever
1078-
// speed up.
1079-
if numGreetings > upperBound {
1080-
t.Fatalf("too many greetings: %d > %d in %v", numGreetings, upperBound, time.Since(start))
1081-
} else if numGreetings < lowerBound {
1082-
t.Fatalf("too few greetings: %d < %d", numGreetings, lowerBound)
1083-
}
1059+
const reportInterval = 2
1060+
const intervalUnit = 100 * time.Millisecond
1061+
var (
1062+
greetingPath = filepath.Join(dir, "greeting")
1063+
script = "echo hello | tee -a " + greetingPath
1064+
)
1065+
_, client, _, _, _ := setupAgent(t, agentsdk.Manifest{
1066+
Metadata: []codersdk.WorkspaceAgentMetadataDescription{
1067+
{
1068+
Key: "greeting",
1069+
Interval: reportInterval,
1070+
Script: script,
1071+
},
1072+
{
1073+
Key: "bad",
1074+
Interval: reportInterval,
1075+
Script: "exit 1",
1076+
},
1077+
},
1078+
}, 0)
1079+
1080+
require.Eventually(t, func() bool {
1081+
return len(client.getMetadata()) == 2
1082+
}, testutil.WaitShort, testutil.IntervalMedium)
1083+
1084+
for start := time.Now(); time.Since(start) < testutil.WaitMedium; time.Sleep(testutil.IntervalMedium) {
1085+
md := client.getMetadata()
1086+
if len(md) != 2 {
1087+
panic("unexpected number of metadata entries")
10841088
}
1085-
})
1089+
1090+
require.Equal(t, "hello\n", md["greeting"].Value)
1091+
require.Equal(t, "exit status 1", md["bad"].Error)
1092+
1093+
greetingByt, err := os.ReadFile(greetingPath)
1094+
require.NoError(t, err)
1095+
1096+
var (
1097+
numGreetings = bytes.Count(greetingByt, []byte("hello"))
1098+
idealNumGreetings = time.Since(start) / (reportInterval * intervalUnit)
1099+
// We allow a 50% error margin because the report loop may backlog
1100+
// in CI and other toasters. In production, there is no hard
1101+
// guarantee on timing either, and the frontend gives similar
1102+
// wiggle room to the staleness of the value.
1103+
upperBound = int(idealNumGreetings) + 1
1104+
lowerBound = (int(idealNumGreetings) / 2)
1105+
)
1106+
1107+
if idealNumGreetings < 50 {
1108+
// There is an insufficient sample size.
1109+
continue
1110+
}
1111+
1112+
t.Logf("numGreetings: %d, idealNumGreetings: %d", numGreetings, idealNumGreetings)
1113+
// The report loop may slow down on load, but it should never, ever
1114+
// speed up.
1115+
if numGreetings > upperBound {
1116+
t.Fatalf("too many greetings: %d > %d in %v", numGreetings, upperBound, time.Since(start))
1117+
} else if numGreetings < lowerBound {
1118+
t.Fatalf("too few greetings: %d < %d", numGreetings, lowerBound)
1119+
}
1120+
}
10861121
}
10871122

10881123
func TestAgent_Lifecycle(t *testing.T) {

‎testutil/timing.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package testutil
2+
3+
import (
4+
"flag"
5+
"testing"
6+
)
7+
8+
// We can't run timing-sensitive tests in CI because of the
9+
// great variance in runner performance. Instead of not testing timing at all,
10+
// we relegate it to humans manually running certain tests with the "-timing"
11+
// flag from time to time.
12+
//
13+
// Eventually, we should run all timing tests in a self-hosted runner.
14+
15+
var timingFlag = flag.Bool("timing", false, "run timing-sensitive tests")
16+
17+
func SkipIfNotTiming(t *testing.T) {
18+
if !*timingFlag {
19+
t.Skip("skipping timing-sensitive test")
20+
}
21+
}

0 commit comments

Comments
 (0)