From 4ed40699cfd4eb2b18d534d1e3d49f61237a6c4a Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 30 Aug 2023 21:46:15 +0000 Subject: [PATCH 01/21] feat: implement agent process management - An opt-in feature has been added to the agent to allow deprioritizing non coder-related processes for both CPU and memory. Non coder processes have their niceness set to 10 and their oom_score_adj set to 100 --- agent/agent.go | 85 +++++++++++++++++++++++++++++++ agent/agentproc/doc.go | 3 ++ agent/agentproc/proc.go | 99 ++++++++++++++++++++++++++++++++++++ agent/agentproc/proc_test.go | 12 +++++ 4 files changed, 199 insertions(+) create mode 100644 agent/agentproc/doc.go create mode 100644 agent/agentproc/proc.go create mode 100644 agent/agentproc/proc_test.go diff --git a/agent/agent.go b/agent/agent.go index 44f55fcedc83d..59f9710723aa5 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -15,6 +15,7 @@ import ( "os/exec" "os/user" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -34,6 +35,7 @@ import ( "tailscale.com/types/netlogtype" "cdr.dev/slog" + "github.com/coder/coder/v2/agent/agentproc" "github.com/coder/coder/v2/agent/agentssh" "github.com/coder/coder/v2/agent/reconnectingpty" "github.com/coder/coder/v2/buildinfo" @@ -51,6 +53,8 @@ const ( ProtocolDial = "dial" ) +const EnvProcMemNice = "CODER_PROC_MEMNICE_ENABLE" + type Options struct { Filesystem afero.Fs LogDir string @@ -68,6 +72,7 @@ type Options struct { PrometheusRegistry *prometheus.Registry ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration + Syscaller agentproc.Syscaller } type Client interface { @@ -197,6 +202,7 @@ type agent struct { prometheusRegistry *prometheus.Registry metrics *agentMetrics + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -225,6 +231,7 @@ func (a *agent) runLoop(ctx context.Context) { go a.reportLifecycleLoop(ctx) go a.reportMetadataLoop(ctx) go a.fetchServiceBannerLoop(ctx) + go a.manageProcessPriorityLoop(ctx) for retrier := retry.New(100*time.Millisecond, 10*time.Second); retrier.Wait(ctx); { a.logger.Info(ctx, "connecting to coderd") @@ -1253,6 +1260,84 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } +var exemptProcesses = []string{"coder"} + +func (a *agent) manageProcessPriorityLoop(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + const ( + procDir = agentproc.DefaultProcDir + niceness = 10 + oomScoreAdj = -1000 + ) + + if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { + a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", + slog.F("env_var", EnvProcMemNice), + slog.F("value", val), + slog.F("goos", runtime.GOOS), + ) + return + } + + for { + select { + case <-ticker.C: + procs, err := agentproc.List(a.filesystem, agentproc.DefaultProcDir) + if err != nil { + a.logger.Error(ctx, "failed to list procs", + slog.F("dir", agentproc.DefaultProcDir), + slog.Error(err), + ) + continue + } + for _, proc := range procs { + // Trim off the path e.g. "./coder" -> "coder" + name := filepath.Base(proc.Name()) + if slices.Contains(exemptProcesses, name) { + a.logger.Debug(ctx, "skipping exempt process", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + ) + continue + } + + err := proc.SetNiceness(a.syscaller, niceness) + if err != nil { + a.logger.Error(ctx, "unable to set proc niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + slog.Error(err), + ) + continue + } + + err = proc.SetOOMAdj(oomScoreAdj) + if err != nil { + a.logger.Error(ctx, "unable to set proc oom_score_adj", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), + slog.Error(err), + ) + continue + } + + a.logger.Debug(ctx, "deprioritized process", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + slog.F("oom_score_adj", oomScoreAdj), + ) + } + case <-ctx.Done(): + return + } + } +} + // isClosed returns whether the API is closed or not. func (a *agent) isClosed() bool { select { diff --git a/agent/agentproc/doc.go b/agent/agentproc/doc.go new file mode 100644 index 0000000000000..8b15c52c5f9fb --- /dev/null +++ b/agent/agentproc/doc.go @@ -0,0 +1,3 @@ +// Package agentproc contains logic for interfacing with local +// processes running in the same context as the agent. +package agentproc diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go new file mode 100644 index 0000000000000..f8b03a8ae4960 --- /dev/null +++ b/agent/agentproc/proc.go @@ -0,0 +1,99 @@ +package agentproc + +import ( + "path/filepath" + "strconv" + "strings" + "syscall" + + "github.com/spf13/afero" + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +const DefaultProcDir = "/proc" + +type Syscaller interface { + SetPriority(pid int32, priority int) error +} + +type UnixSyscaller struct{} + +func (UnixSyscaller) SetPriority(pid int32, nice int) error { + err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) + if err != nil { + return xerrors.Errorf("set priority: %w", err) + } + return nil +} + +type Process struct { + Dir string + CmdLine string + PID int32 + fs afero.Fs +} + +func (p *Process) SetOOMAdj(score int) error { + path := filepath.Join(p.Dir, "oom_score_adj") + err := afero.WriteFile(p.fs, + path, + []byte(strconv.Itoa(score)), + 0644, + ) + if err != nil { + return xerrors.Errorf("write %q: %w", path, err) + } + + return nil +} + +func (p *Process) SetNiceness(sc Syscaller, score int) error { + err := sc.SetPriority(p.PID, score) + if err != nil { + return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) + } + return nil +} + +func (p *Process) Name() string { + args := strings.Split(p.CmdLine, "\x00") + // Split will always return at least one element. + return args[0] +} + +func List(fs afero.Fs, dir string) ([]*Process, error) { + d, err := fs.Open(dir) + if err != nil { + return nil, xerrors.Errorf("open dir %q: %w", dir, err) + } + + entries, err := d.Readdirnames(0) + if err != nil { + return nil, xerrors.Errorf("readdirnames: %w", err) + } + + processes := make([]*Process, 0, len(entries)) + for _, entry := range entries { + pid, err := strconv.ParseInt(entry, 10, 32) + if err != nil { + continue + } + cmdline, err := afero.ReadFile(fs, filepath.Join(dir, entry, "cmdline")) + if err != nil { + var errNo syscall.Errno + if xerrors.As(err, &errNo) && errNo == syscall.EPERM { + continue + } + return nil, xerrors.Errorf("read cmdline: %w", err) + } + processes = append(processes, &Process{ + PID: int32(pid), + CmdLine: string(cmdline), + Dir: filepath.Join(dir, entry), + fs: fs, + }) + } + + return processes, nil +} diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go new file mode 100644 index 0000000000000..42ee8cd7fa140 --- /dev/null +++ b/agent/agentproc/proc_test.go @@ -0,0 +1,12 @@ +package agentproc_test + +type mockSyscaller struct { + SetPriorityFn func(int32, int) error +} + +func (f mockSyscaller) SetPriority(pid int32, nice int) error { + if f.SetPriorityFn == nil { + return nil + } + return f.SetPriorityFn(pid, nice) +} From 7e59db6ed7af22500be7dcc9a4f243cbabe43858 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 31 Aug 2023 00:10:34 +0000 Subject: [PATCH 02/21] improve process detection --- agent/agent.go | 44 +++++++++++++++++++++-------- agent/agentproc/proc.go | 62 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 59f9710723aa5..8f74ea8c2d4ce 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1260,7 +1260,7 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } -var exemptProcesses = []string{"coder"} +var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { ticker := time.NewTicker(time.Minute) @@ -1269,7 +1269,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { const ( procDir = agentproc.DefaultProcDir niceness = 10 - oomScoreAdj = -1000 + oomScoreAdj = 100 ) if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { @@ -1284,7 +1284,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { for { select { case <-ticker.C: - procs, err := agentproc.List(a.filesystem, agentproc.DefaultProcDir) + procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) if err != nil { a.logger.Error(ctx, "failed to list procs", slog.F("dir", agentproc.DefaultProcDir), @@ -1295,31 +1295,52 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { for _, proc := range procs { // Trim off the path e.g. "./coder" -> "coder" name := filepath.Base(proc.Name()) - if slices.Contains(exemptProcesses, name) { - a.logger.Debug(ctx, "skipping exempt process", + // If the process is prioritized we should adjust + // it's oom_score_adj and avoid lowering its niceness. + if slices.Contains(prioritizedProcs, name) { + err = proc.SetOOMAdj(oomScoreAdj) + if err != nil { + a.logger.Error(ctx, "unable to set proc oom_score_adj", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), + slog.Error(err), + ) + continue + } + + a.logger.Debug(ctx, "decreased process oom_score", slog.F("name", proc.Name()), slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), ) continue } - err := proc.SetNiceness(a.syscaller, niceness) + score, err := proc.Nice(a.syscaller) if err != nil { - a.logger.Error(ctx, "unable to set proc niceness", + a.logger.Error(ctx, "unable to get proc niceness", slog.F("name", proc.Name()), slog.F("pid", proc.PID), - slog.F("niceness", niceness), slog.Error(err), ) continue } + if score != 20 { + a.logger.Error(ctx, "skipping process due to custom niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", score), + ) + continue + } - err = proc.SetOOMAdj(oomScoreAdj) + err = proc.SetNiceness(a.syscaller, niceness) if err != nil { - a.logger.Error(ctx, "unable to set proc oom_score_adj", + a.logger.Error(ctx, "unable to set proc niceness", slog.F("name", proc.Name()), slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), + slog.F("niceness", niceness), slog.Error(err), ) continue @@ -1329,7 +1350,6 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { slog.F("name", proc.Name()), slog.F("pid", proc.PID), slog.F("niceness", niceness), - slog.F("oom_score_adj", oomScoreAdj), ) } case <-ctx.Done(): diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go index f8b03a8ae4960..de367db7a4f90 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc.go @@ -15,6 +15,8 @@ const DefaultProcDir = "/proc" type Syscaller interface { SetPriority(pid int32, priority int) error + GetPriority(pid int32) (int, error) + Kill(pid int32, sig syscall.Signal) error } type UnixSyscaller struct{} @@ -27,6 +29,23 @@ func (UnixSyscaller) SetPriority(pid int32, nice int) error { return nil } +func (UnixSyscaller) GetPriority(pid int32) (int, error) { + nice, err := unix.Getpriority(0, int(pid)) + if err != nil { + return 0, xerrors.Errorf("get priority: %w", err) + } + return nice, nil +} + +func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { + err := syscall.Kill(pid, sig) + if err != nil { + return xerrors.Errorf("kill: %w", err) + } + + return nil +} + type Process struct { Dir string CmdLine string @@ -56,13 +75,21 @@ func (p *Process) SetNiceness(sc Syscaller, score int) error { return nil } +func (p *Process) Nice(sc Syscaller) (int, error) { + nice, err := sc.GetPriority(p.PID) + if err != nil { + return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) + } + return nice, nil +} + func (p *Process) Name() string { args := strings.Split(p.CmdLine, "\x00") // Split will always return at least one element. return args[0] } -func List(fs afero.Fs, dir string) ([]*Process, error) { +func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { d, err := fs.Open(dir) if err != nil { return nil, xerrors.Errorf("open dir %q: %w", dir, err) @@ -79,6 +106,16 @@ func List(fs afero.Fs, dir string) ([]*Process, error) { if err != nil { continue } + + // Check that the process still exists. + exists, err := isProcessExist(syscaller, int32(pid), syscall.Signal(0)) + if err != nil { + return nil, xerrors.Errorf("check process exists: %w", err) + } + if !exists { + continue + } + cmdline, err := afero.ReadFile(fs, filepath.Join(dir, entry, "cmdline")) if err != nil { var errNo syscall.Errno @@ -97,3 +134,26 @@ func List(fs afero.Fs, dir string) ([]*Process, error) { return processes, nil } + +func isProcessExist(syscaller Syscaller, pid int32, sig syscall.Signal) (bool, error) { + err := syscaller.Kill(pid, sig) + if err == nil { + return true, nil + } + if err.Error() == "os: process already finished" { + return false, nil + } + + errno, ok := err.(syscall.Errno) + if !ok { + return false, err + } + switch errno { + case syscall.ESRCH: + return false, nil + case syscall.EPERM: + return true, nil + } + + return false, xerrors.Errorf("kill: %w", err) +} From 8c652162f3c30ecbb71cae6607242b6b0388ab49 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 31 Aug 2023 21:47:04 +0000 Subject: [PATCH 03/21] add agentproc tests --- agent/agent.go | 2 +- agent/agentproc/agentproctest/proc.go | 44 +++++++ agent/agentproc/doc.go | 2 + agent/agentproc/proc.go | 68 +++------- agent/agentproc/proc_test.go | 182 +++++++++++++++++++++++++- agent/agentproc/syscaller.go | 41 ++++++ agent/agentproc/syscallermock_test.go | 78 +++++++++++ 7 files changed, 359 insertions(+), 58 deletions(-) create mode 100644 agent/agentproc/agentproctest/proc.go create mode 100644 agent/agentproc/syscaller.go create mode 100644 agent/agentproc/syscallermock_test.go diff --git a/agent/agent.go b/agent/agent.go index 8f74ea8c2d4ce..1f270c4123f0c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1317,7 +1317,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { continue } - score, err := proc.Nice(a.syscaller) + score, err := proc.Niceness(a.syscaller) if err != nil { a.logger.Error(ctx, "unable to get proc niceness", slog.F("name", proc.Name()), diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go new file mode 100644 index 0000000000000..1ef599c2bf85b --- /dev/null +++ b/agent/agentproc/agentproctest/proc.go @@ -0,0 +1,44 @@ +package agentproctest + +import ( + "fmt" + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/agent/agentproc" + "github.com/coder/coder/v2/cryptorand" +) + +func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { + t.Helper() + + pid, err := cryptorand.Intn(1<<31 - 1) + require.NoError(t, err) + + err = fs.MkdirAll(fmt.Sprintf("/%s/%d", dir, pid), 0555) + require.NoError(t, err) + + arg1, err := cryptorand.String(5) + require.NoError(t, err) + + arg2, err := cryptorand.String(5) + require.NoError(t, err) + + arg3, err := cryptorand.String(5) + require.NoError(t, err) + + cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3) + + err = afero.WriteFile(fs, fmt.Sprintf("/%s/%d/cmdline", dir, pid), []byte(cmdline), 0444) + require.NoError(t, err) + + return agentproc.Process{ + PID: int32(pid), + CmdLine: cmdline, + Dir: fmt.Sprintf("%s/%d", dir, pid), + FS: fs, + } + +} diff --git a/agent/agentproc/doc.go b/agent/agentproc/doc.go index 8b15c52c5f9fb..6f65c6ba99d53 100644 --- a/agent/agentproc/doc.go +++ b/agent/agentproc/doc.go @@ -1,3 +1,5 @@ // Package agentproc contains logic for interfacing with local // processes running in the same context as the agent. package agentproc + +//go:generate mockgen -destination ./syscallermock_test.go -package agentproc_test github.com/coder/coder/v2/agent/agentproc Syscaller diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go index de367db7a4f90..eaf213eb88faa 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc.go @@ -1,61 +1,28 @@ package agentproc import ( + "errors" "path/filepath" "strconv" "strings" "syscall" "github.com/spf13/afero" - "golang.org/x/sys/unix" "golang.org/x/xerrors" ) const DefaultProcDir = "/proc" -type Syscaller interface { - SetPriority(pid int32, priority int) error - GetPriority(pid int32) (int, error) - Kill(pid int32, sig syscall.Signal) error -} - -type UnixSyscaller struct{} - -func (UnixSyscaller) SetPriority(pid int32, nice int) error { - err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) - if err != nil { - return xerrors.Errorf("set priority: %w", err) - } - return nil -} - -func (UnixSyscaller) GetPriority(pid int32) (int, error) { - nice, err := unix.Getpriority(0, int(pid)) - if err != nil { - return 0, xerrors.Errorf("get priority: %w", err) - } - return nice, nil -} - -func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { - err := syscall.Kill(pid, sig) - if err != nil { - return xerrors.Errorf("kill: %w", err) - } - - return nil -} - type Process struct { Dir string CmdLine string PID int32 - fs afero.Fs + FS afero.Fs } func (p *Process) SetOOMAdj(score int) error { path := filepath.Join(p.Dir, "oom_score_adj") - err := afero.WriteFile(p.fs, + err := afero.WriteFile(p.FS, path, []byte(strconv.Itoa(score)), 0644, @@ -67,20 +34,20 @@ func (p *Process) SetOOMAdj(score int) error { return nil } -func (p *Process) SetNiceness(sc Syscaller, score int) error { - err := sc.SetPriority(p.PID, score) +func (p *Process) Niceness(sc Syscaller) (int, error) { + nice, err := sc.GetPriority(p.PID) if err != nil { - return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) + return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) } - return nil + return nice, nil } -func (p *Process) Nice(sc Syscaller) (int, error) { - nice, err := sc.GetPriority(p.PID) +func (p *Process) SetNiceness(sc Syscaller, score int) error { + err := sc.SetPriority(p.PID, score) if err != nil { - return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) + return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) } - return nice, nil + return nil } func (p *Process) Name() string { @@ -108,7 +75,7 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { } // Check that the process still exists. - exists, err := isProcessExist(syscaller, int32(pid), syscall.Signal(0)) + exists, err := isProcessExist(syscaller, int32(pid)) if err != nil { return nil, xerrors.Errorf("check process exists: %w", err) } @@ -128,15 +95,15 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { PID: int32(pid), CmdLine: string(cmdline), Dir: filepath.Join(dir, entry), - fs: fs, + FS: fs, }) } return processes, nil } -func isProcessExist(syscaller Syscaller, pid int32, sig syscall.Signal) (bool, error) { - err := syscaller.Kill(pid, sig) +func isProcessExist(syscaller Syscaller, pid int32) (bool, error) { + err := syscaller.Kill(pid, syscall.Signal(0)) if err == nil { return true, nil } @@ -144,10 +111,11 @@ func isProcessExist(syscaller Syscaller, pid int32, sig syscall.Signal) (bool, e return false, nil } - errno, ok := err.(syscall.Errno) - if !ok { + var errno syscall.Errno + if !errors.As(err, &errno) { return false, err } + switch errno { case syscall.ESRCH: return false, nil diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 42ee8cd7fa140..0534d655f4464 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -1,12 +1,180 @@ package agentproc_test -type mockSyscaller struct { - SetPriorityFn func(int32, int) error +import ( + "fmt" + "strings" + "syscall" + "testing" + + "github.com/golang/mock/gomock" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/agent/agentproc" + "github.com/coder/coder/v2/agent/agentproc/agentproctest" +) + +func TestList(t *testing.T) { + t.Parallel() + + t.Run("OK", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + sc = NewMockSyscaller(gomock.NewController(t)) + expectedProcs = make(map[int32]agentproc.Process) + rootDir = agentproc.DefaultProcDir + ) + + for i := 0; i < 4; i++ { + proc := agentproctest.GenerateProcess(t, fs, rootDir) + expectedProcs[proc.PID] = proc + + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + } + + actualProcs, err := agentproc.List(fs, sc, rootDir) + require.NoError(t, err) + require.Len(t, actualProcs, 4) + for _, proc := range actualProcs { + expected, ok := expectedProcs[proc.PID] + require.True(t, ok) + require.Equal(t, expected.PID, proc.PID) + require.Equal(t, expected.CmdLine, proc.CmdLine) + require.Equal(t, expected.Dir, proc.Dir) + } + }) + + t.Run("FinishedProcess", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + sc = NewMockSyscaller(gomock.NewController(t)) + expectedProcs = make(map[int32]agentproc.Process) + rootDir = agentproc.DefaultProcDir + ) + + for i := 0; i < 3; i++ { + proc := agentproctest.GenerateProcess(t, fs, rootDir) + expectedProcs[proc.PID] = proc + + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + } + + // Create a process that's already finished. We're not adding + // it to the map because it should be skipped over. + proc := agentproctest.GenerateProcess(t, fs, rootDir) + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(xerrors.New("os: process already finished")) + + actualProcs, err := agentproc.List(fs, sc, rootDir) + require.NoError(t, err) + require.Len(t, actualProcs, 3) + for _, proc := range actualProcs { + expected, ok := expectedProcs[proc.PID] + require.True(t, ok) + require.Equal(t, expected.PID, proc.PID) + require.Equal(t, expected.CmdLine, proc.CmdLine) + require.Equal(t, expected.Dir, proc.Dir) + } + }) + + t.Run("NoSuchProcess", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + sc = NewMockSyscaller(gomock.NewController(t)) + expectedProcs = make(map[int32]agentproc.Process) + rootDir = agentproc.DefaultProcDir + ) + + for i := 0; i < 3; i++ { + proc := agentproctest.GenerateProcess(t, fs, rootDir) + expectedProcs[proc.PID] = proc + + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + } + + // Create a process that doesn't exist. We're not adding + // it to the map because it should be skipped over. + proc := agentproctest.GenerateProcess(t, fs, rootDir) + sc.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(syscall.ESRCH) + + actualProcs, err := agentproc.List(fs, sc, rootDir) + require.NoError(t, err) + require.Len(t, actualProcs, 3) + for _, proc := range actualProcs { + expected, ok := expectedProcs[proc.PID] + require.True(t, ok) + require.Equal(t, expected.PID, proc.PID) + require.Equal(t, expected.CmdLine, proc.CmdLine) + require.Equal(t, expected.Dir, proc.Dir) + } + }) } -func (f mockSyscaller) SetPriority(pid int32, nice int) error { - if f.SetPriorityFn == nil { - return nil - } - return f.SetPriorityFn(pid, nice) +// These tests are not very interesting but they provide some modicum of +// confidence. +func TestProcess(t *testing.T) { + t.Parallel() + + t.Run("SetOOMAdj", func(t *testing.T) { + t.Parallel() + + var ( + fs = afero.NewMemMapFs() + dir = agentproc.DefaultProcDir + proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + expectedScore = -1000 + ) + + err := proc.SetOOMAdj(expectedScore) + require.NoError(t, err) + + actualScore, err := afero.ReadFile(fs, fmt.Sprintf("%s/%d/oom_score_adj", dir, proc.PID)) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("%d", expectedScore), strings.TrimSpace(string(actualScore))) + }) + + t.Run("SetNiceness", func(t *testing.T) { + t.Parallel() + + var ( + sc = NewMockSyscaller(gomock.NewController(t)) + proc = &agentproc.Process{ + PID: 32, + } + score = 20 + ) + + sc.EXPECT().SetPriority(proc.PID, score).Return(nil) + err := proc.SetNiceness(sc, score) + require.NoError(t, err) + }) + + t.Run("Name", func(t *testing.T) { + t.Parallel() + + var ( + proc = &agentproc.Process{ + CmdLine: "helloworld\x00--arg1\x00--arg2", + } + expectedName = "helloworld" + ) + + require.Equal(t, expectedName, proc.Name()) + }) } diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go new file mode 100644 index 0000000000000..d449e159e0b65 --- /dev/null +++ b/agent/agentproc/syscaller.go @@ -0,0 +1,41 @@ +package agentproc + +import ( + "syscall" + + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +type Syscaller interface { + SetPriority(pid int32, priority int) error + GetPriority(pid int32) (int, error) + Kill(pid int32, sig syscall.Signal) error +} + +type UnixSyscaller struct{} + +func (UnixSyscaller) SetPriority(pid int32, nice int) error { + err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) + if err != nil { + return xerrors.Errorf("set priority: %w", err) + } + return nil +} + +func (UnixSyscaller) GetPriority(pid int32) (int, error) { + nice, err := unix.Getpriority(0, int(pid)) + if err != nil { + return 0, xerrors.Errorf("get priority: %w", err) + } + return nice, nil +} + +func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { + err := syscall.Kill(pid, sig) + if err != nil { + return xerrors.Errorf("kill: %w", err) + } + + return nil +} diff --git a/agent/agentproc/syscallermock_test.go b/agent/agentproc/syscallermock_test.go new file mode 100644 index 0000000000000..e2a250fbe5736 --- /dev/null +++ b/agent/agentproc/syscallermock_test.go @@ -0,0 +1,78 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller) + +// Package agentproc_test is a generated GoMock package. +package agentproc_test + +import ( + reflect "reflect" + syscall "syscall" + + gomock "github.com/golang/mock/gomock" +) + +// MockSyscaller is a mock of Syscaller interface. +type MockSyscaller struct { + ctrl *gomock.Controller + recorder *MockSyscallerMockRecorder +} + +// MockSyscallerMockRecorder is the mock recorder for MockSyscaller. +type MockSyscallerMockRecorder struct { + mock *MockSyscaller +} + +// NewMockSyscaller creates a new mock instance. +func NewMockSyscaller(ctrl *gomock.Controller) *MockSyscaller { + mock := &MockSyscaller{ctrl: ctrl} + mock.recorder = &MockSyscallerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSyscaller) EXPECT() *MockSyscallerMockRecorder { + return m.recorder +} + +// GetPriority mocks base method. +func (m *MockSyscaller) GetPriority(arg0 int32) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPriority", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetPriority indicates an expected call of GetPriority. +func (mr *MockSyscallerMockRecorder) GetPriority(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPriority", reflect.TypeOf((*MockSyscaller)(nil).GetPriority), arg0) +} + +// Kill mocks base method. +func (m *MockSyscaller) Kill(arg0 int32, arg1 syscall.Signal) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Kill", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Kill indicates an expected call of Kill. +func (mr *MockSyscallerMockRecorder) Kill(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kill", reflect.TypeOf((*MockSyscaller)(nil).Kill), arg0, arg1) +} + +// SetPriority mocks base method. +func (m *MockSyscaller) SetPriority(arg0 int32, arg1 int) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetPriority", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetPriority indicates an expected call of SetPriority. +func (mr *MockSyscallerMockRecorder) SetPriority(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPriority", reflect.TypeOf((*MockSyscaller)(nil).SetPriority), arg0, arg1) +} From 760cbb38d56bb7f76564cc4e42a6162eb8a14bd3 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 1 Sep 2023 00:11:48 +0000 Subject: [PATCH 04/21] some minor agent tests --- agent/agent.go | 164 +++++++++++++++----------- agent/agent_test.go | 43 +++++++ agent/agentproc/agentproctest/proc.go | 1 - agent/agentproc/syscaller.go | 4 +- 4 files changed, 141 insertions(+), 71 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 1f270c4123f0c..8c7c7d693e5a8 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -73,6 +73,7 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller + ProcessManagementInterval time.Duration } type Client interface { @@ -125,6 +126,14 @@ func New(options Options) Agent { prometheusRegistry = prometheus.NewRegistry() } + if options.Syscaller == nil { + options.Syscaller = agentproc.UnixSyscaller{} + } + + if options.ProcessManagementInterval == 0 { + options.ProcessManagementInterval = time.Second + } + ctx, cancelFunc := context.WithCancel(context.Background()) a := &agent{ tailnetListenPort: options.TailnetListenPort, @@ -148,6 +157,8 @@ func New(options Options) Agent { sshMaxTimeout: options.SSHMaxTimeout, subsystems: options.Subsystems, addresses: options.Addresses, + syscaller: options.Syscaller, + processManagementInterval: options.ProcessManagementInterval, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -200,9 +211,10 @@ type agent struct { connCountReconnectingPTY atomic.Int64 - prometheusRegistry *prometheus.Registry - metrics *agentMetrics - syscaller agentproc.Syscaller + prometheusRegistry *prometheus.Registry + metrics *agentMetrics + processManagementInterval time.Duration + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1263,15 +1275,9 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - ticker := time.NewTicker(time.Minute) + ticker := time.NewTicker(a.processManagementInterval) defer ticker.Stop() - const ( - procDir = agentproc.DefaultProcDir - niceness = 10 - oomScoreAdj = 100 - ) - if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", slog.F("env_var", EnvProcMemNice), @@ -1281,81 +1287,103 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { return } + // Do once before falling into loop. + if err := a.manageProcessPriority(ctx); err != nil { + a.logger.Error(ctx, "manage process priority", + slog.F("dir", agentproc.DefaultProcDir), + slog.Error(err), + ) + } + for { select { case <-ticker.C: - procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) - if err != nil { - a.logger.Error(ctx, "failed to list procs", + if err := a.manageProcessPriority(ctx); err != nil { + a.logger.Error(ctx, "manage process priority", slog.F("dir", agentproc.DefaultProcDir), slog.Error(err), ) - continue } - for _, proc := range procs { - // Trim off the path e.g. "./coder" -> "coder" - name := filepath.Base(proc.Name()) - // If the process is prioritized we should adjust - // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(prioritizedProcs, name) { - err = proc.SetOOMAdj(oomScoreAdj) - if err != nil { - a.logger.Error(ctx, "unable to set proc oom_score_adj", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), - slog.Error(err), - ) - continue - } - a.logger.Debug(ctx, "decreased process oom_score", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), - ) - continue - } + case <-ctx.Done(): + return + } + } +} - score, err := proc.Niceness(a.syscaller) - if err != nil { - a.logger.Error(ctx, "unable to get proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.Error(err), - ) - continue - } - if score != 20 { - a.logger.Error(ctx, "skipping process due to custom niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("niceness", score), - ) - continue - } +func (a *agent) manageProcessPriority(ctx context.Context) error { + const ( + procDir = agentproc.DefaultProcDir + niceness = 10 + oomScoreAdj = 100 + ) - err = proc.SetNiceness(a.syscaller, niceness) - if err != nil { - a.logger.Error(ctx, "unable to set proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("niceness", niceness), - slog.Error(err), - ) - continue - } + procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) + if err != nil { + return xerrors.Errorf("list: %w", err) + } - a.logger.Debug(ctx, "deprioritized process", + for _, proc := range procs { + // Trim off the path e.g. "./coder" -> "coder" + name := filepath.Base(proc.Name()) + // If the process is prioritized we should adjust + // it's oom_score_adj and avoid lowering its niceness. + if slices.Contains(prioritizedProcs, name) { + err = proc.SetOOMAdj(oomScoreAdj) + if err != nil { + a.logger.Error(ctx, "unable to set proc oom_score_adj", slog.F("name", proc.Name()), slog.F("pid", proc.PID), - slog.F("niceness", niceness), + slog.F("oom_score_adj", oomScoreAdj), + slog.Error(err), ) + continue } - case <-ctx.Done(): - return + + a.logger.Debug(ctx, "decreased process oom_score", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("oom_score_adj", oomScoreAdj), + ) + continue } + + score, err := proc.Niceness(a.syscaller) + if err != nil { + a.logger.Error(ctx, "unable to get proc niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.Error(err), + ) + continue + } + if score != 20 { + a.logger.Error(ctx, "skipping process due to custom niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", score), + ) + continue + } + + err = proc.SetNiceness(a.syscaller, niceness) + if err != nil { + a.logger.Error(ctx, "unable to set proc niceness", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + slog.Error(err), + ) + continue + } + + a.logger.Debug(ctx, "deprioritized process", + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + slog.F("niceness", niceness), + ) } + return nil } // isClosed returns whether the API is closed or not. diff --git a/agent/agent_test.go b/agent/agent_test.go index 126e0f4fa4c97..99bad943c6e28 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -41,6 +41,7 @@ import ( "tailscale.com/tailcfg" "cdr.dev/slog" + "cdr.dev/slog/sloggers/sloghuman" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/agent" "github.com/coder/coder/v2/agent/agentssh" @@ -2395,6 +2396,48 @@ func TestAgent_Metrics_SSH(t *testing.T) { require.NoError(t, err) } +func TestAgent_ManageProcessPriority(t *testing.T) { + t.Parallel() + + t.Run("DisabledByDefault", func(t *testing.T) { + t.Parallel() + + if runtime.GOOS != "linux" { + t.Skip("Skipping non-linux environment") + } + + var buf bytes.Buffer + log := slog.Make(sloghuman.Sink(&buf)) + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.Logger = log + }) + + require.Eventually(t, func() bool { + return strings.Contains(buf.String(), "process priority not enabled") + }, testutil.WaitLong, testutil.IntervalFast) + }) + + t.Run("DisabledForNonLinux", func(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "linux" { + t.Skip("Skipping linux environment") + } + + var buf bytes.Buffer + log := slog.Make(sloghuman.Sink(&buf)) + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.Logger = log + o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + }) + require.Eventually(t, func() bool { + return strings.Contains(buf.String(), "process priority not enabled") + }, testutil.WaitLong, testutil.IntervalFast) + }) +} + func verifyCollectedMetrics(t *testing.T, expected []agentsdk.AgentMetric, actual []*promgo.MetricFamily) bool { t.Helper() diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 1ef599c2bf85b..51ed45112fe86 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -40,5 +40,4 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { Dir: fmt.Sprintf("%s/%d", dir, pid), FS: fs, } - } diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index d449e159e0b65..25a067deed67e 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -31,8 +31,8 @@ func (UnixSyscaller) GetPriority(pid int32) (int, error) { return nice, nil } -func (UnixSyscaller) Kill(pid int, sig syscall.Signal) error { - err := syscall.Kill(pid, sig) +func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error { + err := syscall.Kill(int(pid), sig) if err != nil { return xerrors.Errorf("kill: %w", err) } From f4b864ee86769fcdfc47d670b92539f7b999dd26 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 1 Sep 2023 01:38:38 +0000 Subject: [PATCH 05/21] add a proper test for proc management --- agent/agent.go | 63 +++++++++--------- agent/agent_test.go | 64 +++++++++++++++++++ agent/agentproc/agentproctest/doc.go | 5 ++ agent/agentproc/agentproctest/proc.go | 30 ++++++--- .../syscallermock.go} | 4 +- agent/agentproc/doc.go | 2 - agent/agentproc/proc_test.go | 8 +-- cli/agent.go | 10 ++- 8 files changed, 136 insertions(+), 50 deletions(-) create mode 100644 agent/agentproc/agentproctest/doc.go rename agent/agentproc/{syscallermock_test.go => agentproctest/syscallermock.go} (96%) diff --git a/agent/agent.go b/agent/agent.go index 8c7c7d693e5a8..fb4a9a22aff73 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -73,7 +73,8 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller - ProcessManagementInterval time.Duration + ProcessManagementTick <-chan time.Time + ModifiedProcesses chan []*agentproc.Process } type Client interface { @@ -130,10 +131,6 @@ func New(options Options) Agent { options.Syscaller = agentproc.UnixSyscaller{} } - if options.ProcessManagementInterval == 0 { - options.ProcessManagementInterval = time.Second - } - ctx, cancelFunc := context.WithCancel(context.Background()) a := &agent{ tailnetListenPort: options.TailnetListenPort, @@ -158,7 +155,8 @@ func New(options Options) Agent { subsystems: options.Subsystems, addresses: options.Addresses, syscaller: options.Syscaller, - processManagementInterval: options.ProcessManagementInterval, + processManagementTick: options.ProcessManagementTick, + modifiedProcs: options.ModifiedProcesses, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -211,10 +209,11 @@ type agent struct { connCountReconnectingPTY atomic.Int64 - prometheusRegistry *prometheus.Registry - metrics *agentMetrics - processManagementInterval time.Duration - syscaller agentproc.Syscaller + prometheusRegistry *prometheus.Registry + metrics *agentMetrics + processManagementTick <-chan time.Time + modifiedProcs chan []*agentproc.Process + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1275,9 +1274,6 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - ticker := time.NewTicker(a.processManagementInterval) - defer ticker.Stop() - if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", slog.F("env_var", EnvProcMemNice), @@ -1287,42 +1283,45 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { return } - // Do once before falling into loop. - if err := a.manageProcessPriority(ctx); err != nil { - a.logger.Error(ctx, "manage process priority", - slog.F("dir", agentproc.DefaultProcDir), - slog.Error(err), - ) + manage := func() { + // Do once before falling into loop. + procs, err := a.manageProcessPriority(ctx) + if err != nil { + a.logger.Error(ctx, "manage process priority", + slog.F("dir", agentproc.DefaultProcDir), + slog.Error(err), + ) + } + if a.modifiedProcs != nil { + a.modifiedProcs <- procs + } } + manage() + for { select { - case <-ticker.C: - if err := a.manageProcessPriority(ctx); err != nil { - a.logger.Error(ctx, "manage process priority", - slog.F("dir", agentproc.DefaultProcDir), - slog.Error(err), - ) - } - + case <-a.processManagementTick: + manage() case <-ctx.Done(): return } } } -func (a *agent) manageProcessPriority(ctx context.Context) error { +func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process, error) { const ( procDir = agentproc.DefaultProcDir niceness = 10 - oomScoreAdj = 100 + oomScoreAdj = -1000 ) procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) if err != nil { - return xerrors.Errorf("list: %w", err) + return nil, xerrors.Errorf("list: %w", err) } + modProcs := []*agentproc.Process{} for _, proc := range procs { // Trim off the path e.g. "./coder" -> "coder" name := filepath.Base(proc.Name()) @@ -1339,6 +1338,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) error { ) continue } + modProcs = append(modProcs, proc) a.logger.Debug(ctx, "decreased process oom_score", slog.F("name", proc.Name()), @@ -1382,8 +1382,9 @@ func (a *agent) manageProcessPriority(ctx context.Context) error { slog.F("pid", proc.PID), slog.F("niceness", niceness), ) + modProcs = append(modProcs, proc) } - return nil + return modProcs, nil } // isClosed returns whether the API is closed or not. diff --git a/agent/agent_test.go b/agent/agent_test.go index 99bad943c6e28..abc0ce49f64d9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -21,10 +21,12 @@ import ( "strings" "sync" "sync/atomic" + "syscall" "testing" "time" scp "github.com/bramvdbogaerde/go-scp" + "github.com/golang/mock/gomock" "github.com/google/uuid" "github.com/pion/udp" "github.com/pkg/sftp" @@ -44,6 +46,8 @@ import ( "cdr.dev/slog/sloggers/sloghuman" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/agent" + "github.com/coder/coder/v2/agent/agentproc" + "github.com/coder/coder/v2/agent/agentproc/agentproctest" "github.com/coder/coder/v2/agent/agentssh" "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/httpapi" @@ -2399,6 +2403,66 @@ func TestAgent_Metrics_SSH(t *testing.T) { func TestAgent_ManageProcessPriority(t *testing.T) { t.Parallel() + t.Run("OK", func(t *testing.T) { + t.Parallel() + + var ( + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + ticker = make(chan time.Time) + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) + ) + + // Create some processes. + for i := 0; i < 4; i++ { + // Create a prioritized process. + var proc agentproc.Process + if i == 0 { + proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir, + func(p *agentproc.Process) { + p.CmdLine = "./coder\x00agent\x00--no-reap" + p.PID = 1 + }, + ) + } else { + proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) + syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) + } + syscaller.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + + expectedProcs[proc.PID] = proc + } + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.ProcessManagementTick = ticker + o.Syscaller = syscaller + o.ModifiedProcesses = modProcs + o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.Filesystem = fs + o.Logger = logger + }) + actualProcs := <-modProcs + require.Len(t, actualProcs, 4) + + for _, actual := range actualProcs { + expectedScore := "0" + expected, ok := expectedProcs[actual.PID] + require.True(t, ok) + if expected.PID == 1 { + expectedScore = "-1000" + } + + score, err := afero.ReadFile(fs, filepath.Join(actual.Dir, "oom_score_adj")) + require.NoError(t, err) + require.Equal(t, expectedScore, strings.TrimSpace(string(score))) + } + }) + t.Run("DisabledByDefault", func(t *testing.T) { t.Parallel() diff --git a/agent/agentproc/agentproctest/doc.go b/agent/agentproc/agentproctest/doc.go new file mode 100644 index 0000000000000..5007b36268f76 --- /dev/null +++ b/agent/agentproc/agentproctest/doc.go @@ -0,0 +1,5 @@ +// Package agentproctest contains utility functions +// for testing process management in the agent. +package agentproctest + +//go:generate mockgen -destination ./syscallermock.go -package agentproctest github.com/coder/coder/v2/agent/agentproc Syscaller diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 51ed45112fe86..748833addd26f 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -11,15 +11,12 @@ import ( "github.com/coder/coder/v2/cryptorand" ) -func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { +func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentproc.Process)) agentproc.Process { t.Helper() pid, err := cryptorand.Intn(1<<31 - 1) require.NoError(t, err) - err = fs.MkdirAll(fmt.Sprintf("/%s/%d", dir, pid), 0555) - require.NoError(t, err) - arg1, err := cryptorand.String(5) require.NoError(t, err) @@ -31,13 +28,26 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string) agentproc.Process { cmdline := fmt.Sprintf("%s\x00%s\x00%s", arg1, arg2, arg3) - err = afero.WriteFile(fs, fmt.Sprintf("/%s/%d/cmdline", dir, pid), []byte(cmdline), 0444) - require.NoError(t, err) - - return agentproc.Process{ - PID: int32(pid), + process := agentproc.Process{ CmdLine: cmdline, - Dir: fmt.Sprintf("%s/%d", dir, pid), + PID: int32(pid), FS: fs, } + + for _, mut := range muts { + mut(&process) + } + + process.Dir = fmt.Sprintf("%s/%d", dir, process.PID) + + err = fs.MkdirAll(process.Dir, 0555) + require.NoError(t, err) + + err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0444) + require.NoError(t, err) + + err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0444) + require.NoError(t, err) + + return process } diff --git a/agent/agentproc/syscallermock_test.go b/agent/agentproc/agentproctest/syscallermock.go similarity index 96% rename from agent/agentproc/syscallermock_test.go rename to agent/agentproc/agentproctest/syscallermock.go index e2a250fbe5736..8d9697bc559ef 100644 --- a/agent/agentproc/syscallermock_test.go +++ b/agent/agentproc/agentproctest/syscallermock.go @@ -1,8 +1,8 @@ // Code generated by MockGen. DO NOT EDIT. // Source: github.com/coder/coder/v2/agent/agentproc (interfaces: Syscaller) -// Package agentproc_test is a generated GoMock package. -package agentproc_test +// Package agentproctest is a generated GoMock package. +package agentproctest import ( reflect "reflect" diff --git a/agent/agentproc/doc.go b/agent/agentproc/doc.go index 6f65c6ba99d53..8b15c52c5f9fb 100644 --- a/agent/agentproc/doc.go +++ b/agent/agentproc/doc.go @@ -1,5 +1,3 @@ // Package agentproc contains logic for interfacing with local // processes running in the same context as the agent. package agentproc - -//go:generate mockgen -destination ./syscallermock_test.go -package agentproc_test github.com/coder/coder/v2/agent/agentproc Syscaller diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 0534d655f4464..ea27dd4a7ee49 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -23,7 +23,7 @@ func TestList(t *testing.T) { var ( fs = afero.NewMemMapFs() - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) rootDir = agentproc.DefaultProcDir ) @@ -54,7 +54,7 @@ func TestList(t *testing.T) { var ( fs = afero.NewMemMapFs() - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) rootDir = agentproc.DefaultProcDir ) @@ -92,7 +92,7 @@ func TestList(t *testing.T) { var ( fs = afero.NewMemMapFs() - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) rootDir = agentproc.DefaultProcDir ) @@ -153,7 +153,7 @@ func TestProcess(t *testing.T) { t.Parallel() var ( - sc = NewMockSyscaller(gomock.NewController(t)) + sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) proc = &agentproc.Process{ PID: 32, } diff --git a/cli/agent.go b/cli/agent.go index 8b77c057ef31a..5f1874dc57bd5 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -29,6 +29,7 @@ import ( "cdr.dev/slog/sloggers/slogjson" "cdr.dev/slog/sloggers/slogstackdriver" "github.com/coder/coder/v2/agent" + "github.com/coder/coder/v2/agent/agentproc" "github.com/coder/coder/v2/agent/reaper" "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/cli/clibase" @@ -267,6 +268,8 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { subsystems = append(subsystems, subsystem) } + procTicker := time.NewTicker(time.Second) + defer procTicker.Stop() agnt := agent.New(agent.Options{ Client: client, Logger: logger, @@ -290,7 +293,12 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { SSHMaxTimeout: sshMaxTimeout, Subsystems: subsystems, - PrometheusRegistry: prometheusRegistry, + PrometheusRegistry: prometheusRegistry, + ProcessManagementTick: procTicker.C, + Syscaller: agentproc.UnixSyscaller{}, + // Intentionally set this to nil. It's mainly used + // for testing. + ModifiedProcesses: nil, }) prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus") From cbcb854f0cc92d9f2c98ad131610117142e53665 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 1 Sep 2023 01:43:20 +0000 Subject: [PATCH 06/21] custom nice --- agent/agent_test.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/agent/agent_test.go b/agent/agent_test.go index abc0ce49f64d9..ba73034a973a6 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2463,6 +2463,49 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } }) + t.Run("IgnoreCustomNice", func(t *testing.T) { + t.Parallel() + + var ( + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + ticker = make(chan time.Time) + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) + ) + + // Create some processes. + for i := 0; i < 2; i++ { + // Create a prioritized process. + proc := agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + syscaller.EXPECT(). + Kill(proc.PID, syscall.Signal(0)). + Return(nil) + + if i == 0 { + syscaller.EXPECT().GetPriority(proc.PID).Return(25, nil) + } else { + syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) + syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) + } + + expectedProcs[proc.PID] = proc + } + + _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { + o.ProcessManagementTick = ticker + o.Syscaller = syscaller + o.ModifiedProcesses = modProcs + o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.Filesystem = fs + o.Logger = logger + }) + actualProcs := <-modProcs + // We should ignore the process with a custom nice score. + require.Len(t, actualProcs, 1) + }) + t.Run("DisabledByDefault", func(t *testing.T) { t.Parallel() From 8230247e32cec554bbcee9de2c09e785ef8adf6d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 8 Sep 2023 22:21:11 +0000 Subject: [PATCH 07/21] refactor into build files --- agent/agent.go | 27 ++++++++++++------- agent/agent_test.go | 6 +---- agent/agentproc/syscaller.go | 30 --------------------- agent/agentproc/syscaller_other.go | 24 +++++++++++++++++ agent/agentproc/syscaller_unix.go | 42 ++++++++++++++++++++++++++++++ cli/agent.go | 8 +++--- 6 files changed, 88 insertions(+), 49 deletions(-) create mode 100644 agent/agentproc/syscaller_other.go create mode 100644 agent/agentproc/syscaller_unix.go diff --git a/agent/agent.go b/agent/agent.go index fb4a9a22aff73..9d63d4c5a3982 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -53,6 +53,8 @@ const ( ProtocolDial = "dial" ) +// EnvProcMemNice determines whether we attempt to manage +// process CPU and OOM Killer priority. const EnvProcMemNice = "CODER_PROC_MEMNICE_ENABLE" type Options struct { @@ -73,7 +75,6 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller - ProcessManagementTick <-chan time.Time ModifiedProcesses chan []*agentproc.Process } @@ -128,7 +129,7 @@ func New(options Options) Agent { } if options.Syscaller == nil { - options.Syscaller = agentproc.UnixSyscaller{} + options.Syscaller = agentproc.NewSyscaller() } ctx, cancelFunc := context.WithCancel(context.Background()) @@ -155,7 +156,6 @@ func New(options Options) Agent { subsystems: options.Subsystems, addresses: options.Addresses, syscaller: options.Syscaller, - processManagementTick: options.ProcessManagementTick, modifiedProcs: options.ModifiedProcesses, prometheusRegistry: prometheusRegistry, @@ -209,11 +209,10 @@ type agent struct { connCountReconnectingPTY atomic.Int64 - prometheusRegistry *prometheus.Registry - metrics *agentMetrics - processManagementTick <-chan time.Time - modifiedProcs chan []*agentproc.Process - syscaller agentproc.Syscaller + prometheusRegistry *prometheus.Registry + metrics *agentMetrics + modifiedProcs chan []*agentproc.Process + syscaller agentproc.Syscaller } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1299,9 +1298,12 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { manage() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { select { - case <-a.processManagementTick: + case <-ticker.C: manage() case <-ctx.Done(): return @@ -1313,7 +1315,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process const ( procDir = agentproc.DefaultProcDir niceness = 10 - oomScoreAdj = -1000 + oomScoreAdj = -500 ) procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) @@ -1357,6 +1359,11 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process ) continue } + + // We only want processes that don't have a nice value set + // so we don't override user nice values. + // Getpriority actually returns priority for the nice value + // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). if score != 20 { a.logger.Error(ctx, "skipping process due to custom niceness", slog.F("name", proc.Name()), diff --git a/agent/agent_test.go b/agent/agent_test.go index ba73034a973a6..06c7f54c7ea08 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2409,7 +2409,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() - ticker = make(chan time.Time) syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) @@ -2439,7 +2438,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { - o.ProcessManagementTick = ticker o.Syscaller = syscaller o.ModifiedProcesses = modProcs o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} @@ -2454,7 +2452,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expected, ok := expectedProcs[actual.PID] require.True(t, ok) if expected.PID == 1 { - expectedScore = "-1000" + expectedScore = "-500" } score, err := afero.ReadFile(fs, filepath.Join(actual.Dir, "oom_score_adj")) @@ -2469,7 +2467,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() - ticker = make(chan time.Time) syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) @@ -2494,7 +2491,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { - o.ProcessManagementTick = ticker o.Syscaller = syscaller o.ModifiedProcesses = modProcs o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index 25a067deed67e..6d346d1fb92ee 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -2,9 +2,6 @@ package agentproc import ( "syscall" - - "golang.org/x/sys/unix" - "golang.org/x/xerrors" ) type Syscaller interface { @@ -12,30 +9,3 @@ type Syscaller interface { GetPriority(pid int32) (int, error) Kill(pid int32, sig syscall.Signal) error } - -type UnixSyscaller struct{} - -func (UnixSyscaller) SetPriority(pid int32, nice int) error { - err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) - if err != nil { - return xerrors.Errorf("set priority: %w", err) - } - return nil -} - -func (UnixSyscaller) GetPriority(pid int32) (int, error) { - nice, err := unix.Getpriority(0, int(pid)) - if err != nil { - return 0, xerrors.Errorf("get priority: %w", err) - } - return nice, nil -} - -func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error { - err := syscall.Kill(int(pid), sig) - if err != nil { - return xerrors.Errorf("kill: %w", err) - } - - return nil -} diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go new file mode 100644 index 0000000000000..ba949d4ac33a5 --- /dev/null +++ b/agent/agentproc/syscaller_other.go @@ -0,0 +1,24 @@ +//go:build !linux +// +build !linux + +package agentproc + +import "syscall" + +func NewSyscaller() Syscaller { + return nopSyscaller{} +} + +type nopSyscaller struct{} + +func (nopSyscaller) SetPriority(pid int32, priority int) error { + return nil +} + +func (nopSyscaller) GetPriority(pid int32) (int, error) { + return 0, nil +} + +func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error { + return nil +} diff --git a/agent/agentproc/syscaller_unix.go b/agent/agentproc/syscaller_unix.go new file mode 100644 index 0000000000000..e63e56b50f724 --- /dev/null +++ b/agent/agentproc/syscaller_unix.go @@ -0,0 +1,42 @@ +//go:build linux +// +build linux + +package agentproc + +import ( + "syscall" + + "golang.org/x/sys/unix" + "golang.org/x/xerrors" +) + +func NewSyscaller() Syscaller { + return UnixSyscaller{} +} + +type UnixSyscaller struct{} + +func (UnixSyscaller) SetPriority(pid int32, nice int) error { + err := unix.Setpriority(unix.PRIO_PROCESS, int(pid), nice) + if err != nil { + return xerrors.Errorf("set priority: %w", err) + } + return nil +} + +func (UnixSyscaller) GetPriority(pid int32) (int, error) { + nice, err := unix.Getpriority(0, int(pid)) + if err != nil { + return 0, xerrors.Errorf("get priority: %w", err) + } + return nice, nil +} + +func (UnixSyscaller) Kill(pid int32, sig syscall.Signal) error { + err := syscall.Kill(int(pid), sig) + if err != nil { + return xerrors.Errorf("kill: %w", err) + } + + return nil +} diff --git a/cli/agent.go b/cli/agent.go index 5f1874dc57bd5..64693e3803e09 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -287,15 +287,15 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { return resp.SessionToken, nil }, EnvironmentVariables: map[string]string{ - "GIT_ASKPASS": executablePath, + "GIT_ASKPASS": executablePath, + agent.EnvProcMemNice: os.Getenv(agent.EnvProcMemNice), }, IgnorePorts: ignorePorts, SSHMaxTimeout: sshMaxTimeout, Subsystems: subsystems, - PrometheusRegistry: prometheusRegistry, - ProcessManagementTick: procTicker.C, - Syscaller: agentproc.UnixSyscaller{}, + PrometheusRegistry: prometheusRegistry, + Syscaller: agentproc.NewSyscaller(), // Intentionally set this to nil. It's mainly used // for testing. ModifiedProcesses: nil, From 3e1defd13e92a5878bba6bf2277ce5b51fb7441d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 8 Sep 2023 22:45:26 +0000 Subject: [PATCH 08/21] tick tock --- agent/agent.go | 23 +++++++++++++++++------ agent/agent_test.go | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 9d63d4c5a3982..d1bed60525a4b 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -75,7 +75,10 @@ type Options struct { ReportMetadataInterval time.Duration ServiceBannerRefreshInterval time.Duration Syscaller agentproc.Syscaller - ModifiedProcesses chan []*agentproc.Process + // ModifiedProcesses is used for testing process priority management. + ModifiedProcesses chan []*agentproc.Process + // ProcessManagementTick is used for testing process priority management. + ProcessManagementTick <-chan time.Time } type Client interface { @@ -157,6 +160,7 @@ func New(options Options) Agent { addresses: options.Addresses, syscaller: options.Syscaller, modifiedProcs: options.ModifiedProcesses, + processManagementTick: options.ProcessManagementTick, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -211,8 +215,12 @@ type agent struct { prometheusRegistry *prometheus.Registry metrics *agentMetrics - modifiedProcs chan []*agentproc.Process syscaller agentproc.Syscaller + + // podifiedProcs is used for testing process priority management. + modifiedProcs chan []*agentproc.Process + // processManagementTick is used for testing process priority management. + processManagementTick <-chan time.Time } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1283,7 +1291,6 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { } manage := func() { - // Do once before falling into loop. procs, err := a.manageProcessPriority(ctx) if err != nil { a.logger.Error(ctx, "manage process priority", @@ -1296,14 +1303,18 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { } } + // Do once before falling into loop. manage() - ticker := time.NewTicker(time.Second) - defer ticker.Stop() + if a.processManagementTick == nil { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + a.processManagementTick = ticker.C + } for { select { - case <-ticker.C: + case <-a.processManagementTick: manage() case <-ctx.Done(): return diff --git a/agent/agent_test.go b/agent/agent_test.go index 06c7f54c7ea08..2fcfbbefbab4d 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2410,13 +2410,16 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + ticker = make(chan time.Time) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) ) // Create some processes. for i := 0; i < 4; i++ { - // Create a prioritized process. + // Create a prioritized process. This process should + // have it's oom_score_adj set to -500 and its nice + // score should be untouched. var proc agentproc.Process if i == 0 { proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir, @@ -2426,6 +2429,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { }, ) } else { + // The rest are peasants. proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) @@ -2443,6 +2447,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} o.Filesystem = fs o.Logger = logger + o.ProcessManagementTick = ticker }) actualProcs := <-modProcs require.Len(t, actualProcs, 4) @@ -2467,6 +2472,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() + ticker = make(chan time.Time) syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) modProcs = make(chan []*agentproc.Process) logger = slog.Make(sloghuman.Sink(io.Discard)) @@ -2474,13 +2480,14 @@ func TestAgent_ManageProcessPriority(t *testing.T) { // Create some processes. for i := 0; i < 2; i++ { - // Create a prioritized process. proc := agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) syscaller.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(nil) if i == 0 { + // Set a random nice score. This one should not be adjusted by + // our management loop. syscaller.EXPECT().GetPriority(proc.PID).Return(25, nil) } else { syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) @@ -2496,6 +2503,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} o.Filesystem = fs o.Logger = logger + o.ProcessManagementTick = ticker }) actualProcs := <-modProcs // We should ignore the process with a custom nice score. @@ -2533,6 +2541,8 @@ func TestAgent_ManageProcessPriority(t *testing.T) { _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log + // Try to enable it so that we can assert that non-linux + // environments are truly disabled. o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} }) require.Eventually(t, func() bool { From 2fe9c70d9c2296b871f879f48e464fd06c88cce5 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 8 Sep 2023 22:48:25 +0000 Subject: [PATCH 09/21] make fmt --- agent/agentproc/agentproctest/proc.go | 6 +++--- agent/agentproc/proc.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 748833addd26f..473825cd9d699 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -40,13 +40,13 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentp process.Dir = fmt.Sprintf("%s/%d", dir, process.PID) - err = fs.MkdirAll(process.Dir, 0555) + err = fs.MkdirAll(process.Dir, 0o555) require.NoError(t, err) - err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0444) + err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444) require.NoError(t, err) - err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0444) + err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0o444) require.NoError(t, err) return process diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc.go index eaf213eb88faa..29cb9a205bf4f 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc.go @@ -25,7 +25,7 @@ func (p *Process) SetOOMAdj(score int) error { err := afero.WriteFile(p.FS, path, []byte(strconv.Itoa(score)), - 0644, + 0o644, ) if err != nil { return xerrors.Errorf("write %q: %w", path, err) From ef41e9a4bcec19817a760bf6cdf48189e4fe73dc Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 12 Sep 2023 22:38:44 +0000 Subject: [PATCH 10/21] pr comments --- agent/agent.go | 74 +++++++----------- agent/agent_test.go | 16 ++-- agent/agentproc/agentproctest/proc.go | 4 +- agent/agentproc/proc_other.go | 28 +++++++ agent/agentproc/proc_test.go | 30 ++++--- agent/agentproc/{proc.go => proc_unix.go} | 95 +++++++++++------------ agent/agentproc/syscaller.go | 11 +++ agent/agentproc/syscaller_other.go | 14 +++- cli/agent.go | 4 +- 9 files changed, 147 insertions(+), 129 deletions(-) create mode 100644 agent/agentproc/proc_other.go rename agent/agentproc/{proc.go => proc_unix.go} (85%) diff --git a/agent/agent.go b/agent/agent.go index d1bed60525a4b..5886f305d2ca9 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -53,9 +53,9 @@ const ( ProtocolDial = "dial" ) -// EnvProcMemNice determines whether we attempt to manage +// EnvProcPrioMgmt determines whether we attempt to manage // process CPU and OOM Killer priority. -const EnvProcMemNice = "CODER_PROC_MEMNICE_ENABLE" +const EnvProcPrioMgmt = "CODER_PROC_PRIO_MGMT" type Options struct { Filesystem afero.Fs @@ -217,7 +217,7 @@ type agent struct { metrics *agentMetrics syscaller agentproc.Syscaller - // podifiedProcs is used for testing process priority management. + // modifiedProcs is used for testing process priority management. modifiedProcs chan []*agentproc.Process // processManagementTick is used for testing process priority management. processManagementTick <-chan time.Time @@ -1281,41 +1281,34 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - if val := a.envVars[EnvProcMemNice]; val == "" || runtime.GOOS != "linux" { - a.logger.Info(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", - slog.F("env_var", EnvProcMemNice), + if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { + a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", + slog.F("env_var", EnvProcPrioMgmt), slog.F("value", val), slog.F("goos", runtime.GOOS), ) return } - manage := func() { + if a.processManagementTick == nil { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + a.processManagementTick = ticker.C + } + + for { procs, err := a.manageProcessPriority(ctx) if err != nil { a.logger.Error(ctx, "manage process priority", - slog.F("dir", agentproc.DefaultProcDir), slog.Error(err), ) } if a.modifiedProcs != nil { a.modifiedProcs <- procs } - } - - // Do once before falling into loop. - manage() - - if a.processManagementTick == nil { - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - a.processManagementTick = ticker.C - } - for { select { case <-a.processManagementTick: - manage() case <-ctx.Done(): return } @@ -1324,18 +1317,26 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process, error) { const ( - procDir = agentproc.DefaultProcDir niceness = 10 oomScoreAdj = -500 ) - procs, err := agentproc.List(a.filesystem, a.syscaller, agentproc.DefaultProcDir) + procs, err := agentproc.List(a.filesystem, a.syscaller) if err != nil { return nil, xerrors.Errorf("list: %w", err) } - modProcs := []*agentproc.Process{} + var ( + modProcs = []*agentproc.Process{} + logger slog.Logger + ) + for _, proc := range procs { + logger = a.logger.With( + slog.F("name", proc.Name()), + slog.F("pid", proc.PID), + ) + // Trim off the path e.g. "./coder" -> "coder" name := filepath.Base(proc.Name()) // If the process is prioritized we should adjust @@ -1343,29 +1344,19 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process if slices.Contains(prioritizedProcs, name) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { - a.logger.Error(ctx, "unable to set proc oom_score_adj", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Warn(ctx, "unable to set proc oom_score_adj", slog.F("oom_score_adj", oomScoreAdj), slog.Error(err), ) continue } modProcs = append(modProcs, proc) - - a.logger.Debug(ctx, "decreased process oom_score", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("oom_score_adj", oomScoreAdj), - ) continue } score, err := proc.Niceness(a.syscaller) if err != nil { - a.logger.Error(ctx, "unable to get proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Warn(ctx, "unable to get proc niceness", slog.Error(err), ) continue @@ -1376,9 +1367,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process // Getpriority actually returns priority for the nice value // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). if score != 20 { - a.logger.Error(ctx, "skipping process due to custom niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Debug(ctx, "skipping process due to custom niceness", slog.F("niceness", score), ) continue @@ -1386,20 +1375,13 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process err = proc.SetNiceness(a.syscaller, niceness) if err != nil { - a.logger.Error(ctx, "unable to set proc niceness", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), + logger.Warn(ctx, "unable to set proc niceness", slog.F("niceness", niceness), slog.Error(err), ) continue } - a.logger.Debug(ctx, "deprioritized process", - slog.F("name", proc.Name()), - slog.F("pid", proc.PID), - slog.F("niceness", niceness), - ) modProcs = append(modProcs, proc) } return modProcs, nil diff --git a/agent/agent_test.go b/agent/agent_test.go index 2fcfbbefbab4d..c2d20209f4974 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2422,7 +2422,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { // score should be untouched. var proc agentproc.Process if i == 0 { - proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir, + proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" p.PID = 1 @@ -2430,7 +2430,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { ) } else { // The rest are peasants. - proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + proc = agentproctest.GenerateProcess(t, fs) syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2444,7 +2444,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Syscaller = syscaller o.ModifiedProcesses = modProcs - o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker @@ -2480,7 +2480,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { // Create some processes. for i := 0; i < 2; i++ { - proc := agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + proc := agentproctest.GenerateProcess(t, fs) syscaller.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(nil) @@ -2500,7 +2500,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Syscaller = syscaller o.ModifiedProcesses = modProcs - o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker @@ -2518,7 +2518,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)) + log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log @@ -2537,13 +2537,13 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)) + log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log // Try to enable it so that we can assert that non-linux // environments are truly disabled. - o.EnvironmentVariables = map[string]string{agent.EnvProcMemNice: "1"} + o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} }) require.Eventually(t, func() bool { return strings.Contains(buf.String(), "process priority not enabled") diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 473825cd9d699..1fe44ad9eb31b 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -11,7 +11,7 @@ import ( "github.com/coder/coder/v2/cryptorand" ) -func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentproc.Process)) agentproc.Process { +func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process)) agentproc.Process { t.Helper() pid, err := cryptorand.Intn(1<<31 - 1) @@ -38,7 +38,7 @@ func GenerateProcess(t *testing.T, fs afero.Fs, dir string, muts ...func(*agentp mut(&process) } - process.Dir = fmt.Sprintf("%s/%d", dir, process.PID) + process.Dir = fmt.Sprintf("%s/%d", "/proc", process.PID) err = fs.MkdirAll(process.Dir, 0o555) require.NoError(t, err) diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go new file mode 100644 index 0000000000000..b299eac185fd5 --- /dev/null +++ b/agent/agentproc/proc_other.go @@ -0,0 +1,28 @@ +//go:build !linux +// +build !linux + +package agentproc + +import ( + "github.com/spf13/afero" +) + +func (p *Process) SetOOMAdj(score int) error { + return errUnimplimented +} + +func (p *Process) Niceness(sc Syscaller) (int, error) { + return 0, errUnimplimented +} + +func (p *Process) SetNiceness(sc Syscaller, score int) error { + return errUnimplimented +} + +func (p *Process) Name() string { + return "" +} + +func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { + return nil, errUnimplimented +} diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index ea27dd4a7ee49..6459b05f3f7e8 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -25,11 +25,10 @@ func TestList(t *testing.T) { fs = afero.NewMemMapFs() sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) - rootDir = agentproc.DefaultProcDir ) for i := 0; i < 4; i++ { - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) expectedProcs[proc.PID] = proc sc.EXPECT(). @@ -37,9 +36,9 @@ func TestList(t *testing.T) { Return(nil) } - actualProcs, err := agentproc.List(fs, sc, rootDir) + actualProcs, err := agentproc.List(fs, sc) require.NoError(t, err) - require.Len(t, actualProcs, 4) + require.Len(t, actualProcs, len(expectedProcs)) for _, proc := range actualProcs { expected, ok := expectedProcs[proc.PID] require.True(t, ok) @@ -56,11 +55,10 @@ func TestList(t *testing.T) { fs = afero.NewMemMapFs() sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) - rootDir = agentproc.DefaultProcDir ) for i := 0; i < 3; i++ { - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) expectedProcs[proc.PID] = proc sc.EXPECT(). @@ -70,14 +68,14 @@ func TestList(t *testing.T) { // Create a process that's already finished. We're not adding // it to the map because it should be skipped over. - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) sc.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(xerrors.New("os: process already finished")) - actualProcs, err := agentproc.List(fs, sc, rootDir) + actualProcs, err := agentproc.List(fs, sc) require.NoError(t, err) - require.Len(t, actualProcs, 3) + require.Len(t, actualProcs, len(expectedProcs)) for _, proc := range actualProcs { expected, ok := expectedProcs[proc.PID] require.True(t, ok) @@ -94,11 +92,10 @@ func TestList(t *testing.T) { fs = afero.NewMemMapFs() sc = agentproctest.NewMockSyscaller(gomock.NewController(t)) expectedProcs = make(map[int32]agentproc.Process) - rootDir = agentproc.DefaultProcDir ) for i := 0; i < 3; i++ { - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) expectedProcs[proc.PID] = proc sc.EXPECT(). @@ -108,14 +105,14 @@ func TestList(t *testing.T) { // Create a process that doesn't exist. We're not adding // it to the map because it should be skipped over. - proc := agentproctest.GenerateProcess(t, fs, rootDir) + proc := agentproctest.GenerateProcess(t, fs) sc.EXPECT(). Kill(proc.PID, syscall.Signal(0)). Return(syscall.ESRCH) - actualProcs, err := agentproc.List(fs, sc, rootDir) + actualProcs, err := agentproc.List(fs, sc) require.NoError(t, err) - require.Len(t, actualProcs, 3) + require.Len(t, actualProcs, len(expectedProcs)) for _, proc := range actualProcs { expected, ok := expectedProcs[proc.PID] require.True(t, ok) @@ -136,15 +133,14 @@ func TestProcess(t *testing.T) { var ( fs = afero.NewMemMapFs() - dir = agentproc.DefaultProcDir - proc = agentproctest.GenerateProcess(t, fs, agentproc.DefaultProcDir) + proc = agentproctest.GenerateProcess(t, fs) expectedScore = -1000 ) err := proc.SetOOMAdj(expectedScore) require.NoError(t, err) - actualScore, err := afero.ReadFile(fs, fmt.Sprintf("%s/%d/oom_score_adj", dir, proc.PID)) + actualScore, err := afero.ReadFile(fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID)) require.NoError(t, err) require.Equal(t, fmt.Sprintf("%d", expectedScore), strings.TrimSpace(string(actualScore))) }) diff --git a/agent/agentproc/proc.go b/agent/agentproc/proc_unix.go similarity index 85% rename from agent/agentproc/proc.go rename to agent/agentproc/proc_unix.go index 29cb9a205bf4f..462f7b57c987c 100644 --- a/agent/agentproc/proc.go +++ b/agent/agentproc/proc_unix.go @@ -1,3 +1,6 @@ +//go:build linux +// +build linux + package agentproc import ( @@ -11,56 +14,12 @@ import ( "golang.org/x/xerrors" ) -const DefaultProcDir = "/proc" - -type Process struct { - Dir string - CmdLine string - PID int32 - FS afero.Fs -} - -func (p *Process) SetOOMAdj(score int) error { - path := filepath.Join(p.Dir, "oom_score_adj") - err := afero.WriteFile(p.FS, - path, - []byte(strconv.Itoa(score)), - 0o644, - ) - if err != nil { - return xerrors.Errorf("write %q: %w", path, err) - } - - return nil -} - -func (p *Process) Niceness(sc Syscaller) (int, error) { - nice, err := sc.GetPriority(p.PID) - if err != nil { - return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) - } - return nice, nil -} - -func (p *Process) SetNiceness(sc Syscaller, score int) error { - err := sc.SetPriority(p.PID, score) - if err != nil { - return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) - } - return nil -} - -func (p *Process) Name() string { - args := strings.Split(p.CmdLine, "\x00") - // Split will always return at least one element. - return args[0] -} - -func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { - d, err := fs.Open(dir) +func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { + d, err := fs.Open(defaultProcDir) if err != nil { - return nil, xerrors.Errorf("open dir %q: %w", dir, err) + return nil, xerrors.Errorf("open dir %q: %w", defaultProcDir, err) } + defer d.Close() entries, err := d.Readdirnames(0) if err != nil { @@ -83,7 +42,7 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { continue } - cmdline, err := afero.ReadFile(fs, filepath.Join(dir, entry, "cmdline")) + cmdline, err := afero.ReadFile(fs, filepath.Join(defaultProcDir, entry, "cmdline")) if err != nil { var errNo syscall.Errno if xerrors.As(err, &errNo) && errNo == syscall.EPERM { @@ -94,7 +53,7 @@ func List(fs afero.Fs, syscaller Syscaller, dir string) ([]*Process, error) { processes = append(processes, &Process{ PID: int32(pid), CmdLine: string(cmdline), - Dir: filepath.Join(dir, entry), + Dir: filepath.Join(defaultProcDir, entry), FS: fs, }) } @@ -125,3 +84,39 @@ func isProcessExist(syscaller Syscaller, pid int32) (bool, error) { return false, xerrors.Errorf("kill: %w", err) } + +func (p *Process) SetOOMAdj(score int) error { + path := filepath.Join(p.Dir, "oom_score_adj") + err := afero.WriteFile(p.FS, + path, + []byte(strconv.Itoa(score)), + 0o644, + ) + if err != nil { + return xerrors.Errorf("write %q: %w", path, err) + } + + return nil +} + +func (p *Process) Niceness(sc Syscaller) (int, error) { + nice, err := sc.GetPriority(p.PID) + if err != nil { + return 0, xerrors.Errorf("get priority for %q: %w", p.CmdLine, err) + } + return nice, nil +} + +func (p *Process) SetNiceness(sc Syscaller, score int) error { + err := sc.SetPriority(p.PID, score) + if err != nil { + return xerrors.Errorf("set priority for %q: %w", p.CmdLine, err) + } + return nil +} + +func (p *Process) Name() string { + args := strings.Split(p.CmdLine, "\x00") + // Split will always return at least one element. + return args[0] +} diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index 6d346d1fb92ee..cf2c90d7ec77d 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -2,6 +2,8 @@ package agentproc import ( "syscall" + + "github.com/spf13/afero" ) type Syscaller interface { @@ -9,3 +11,12 @@ type Syscaller interface { GetPriority(pid int32) (int, error) Kill(pid int32, sig syscall.Signal) error } + +const defaultProcDir = "/proc" + +type Process struct { + Dir string + CmdLine string + PID int32 + FS afero.Fs +} diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go index ba949d4ac33a5..5177cac2ddfd1 100644 --- a/agent/agentproc/syscaller_other.go +++ b/agent/agentproc/syscaller_other.go @@ -3,22 +3,28 @@ package agentproc -import "syscall" +import ( + "syscall" + + "golang.org/x/xerrors" +) func NewSyscaller() Syscaller { return nopSyscaller{} } +var errUnimplimented = xerrors.New("unimplemented") + type nopSyscaller struct{} func (nopSyscaller) SetPriority(pid int32, priority int) error { - return nil + return errUnimplimented } func (nopSyscaller) GetPriority(pid int32) (int, error) { - return 0, nil + return 0, errUnimplimented } func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error { - return nil + return errUnimplimented } diff --git a/cli/agent.go b/cli/agent.go index 64693e3803e09..6a06f4d454f91 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -287,8 +287,8 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { return resp.SessionToken, nil }, EnvironmentVariables: map[string]string{ - "GIT_ASKPASS": executablePath, - agent.EnvProcMemNice: os.Getenv(agent.EnvProcMemNice), + "GIT_ASKPASS": executablePath, + agent.EnvProcPrioMgmt: os.Getenv(agent.EnvProcPrioMgmt), }, IgnorePorts: ignorePorts, SSHMaxTimeout: sshMaxTimeout, From 05baba0382a048e7d385e6328f5f10847ceec69e Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 12 Sep 2023 22:44:54 +0000 Subject: [PATCH 11/21] lint --- agent/agentproc/proc_other.go | 8 ++++---- agent/agentproc/syscaller_other.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go index b299eac185fd5..b4d6348003572 100644 --- a/agent/agentproc/proc_other.go +++ b/agent/agentproc/proc_other.go @@ -8,15 +8,15 @@ import ( ) func (p *Process) SetOOMAdj(score int) error { - return errUnimplimented + return errUnimplemented } func (p *Process) Niceness(sc Syscaller) (int, error) { - return 0, errUnimplimented + return 0, errUnimplemented } func (p *Process) SetNiceness(sc Syscaller, score int) error { - return errUnimplimented + return errUnimplemented } func (p *Process) Name() string { @@ -24,5 +24,5 @@ func (p *Process) Name() string { } func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { - return nil, errUnimplimented + return nil, errUnimplemented } diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go index 5177cac2ddfd1..2b61910704fd8 100644 --- a/agent/agentproc/syscaller_other.go +++ b/agent/agentproc/syscaller_other.go @@ -13,7 +13,7 @@ func NewSyscaller() Syscaller { return nopSyscaller{} } -var errUnimplimented = xerrors.New("unimplemented") +var errUnimplemented = xerrors.New("unimplemented") type nopSyscaller struct{} From 478d57c51fc5879708c89df5af949a28fb038311 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 12 Sep 2023 23:04:39 +0000 Subject: [PATCH 12/21] whoops --- agent/agentproc/syscaller_other.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/agent/agentproc/syscaller_other.go b/agent/agentproc/syscaller_other.go index 2b61910704fd8..114c553e43da2 100644 --- a/agent/agentproc/syscaller_other.go +++ b/agent/agentproc/syscaller_other.go @@ -18,13 +18,13 @@ var errUnimplemented = xerrors.New("unimplemented") type nopSyscaller struct{} func (nopSyscaller) SetPriority(pid int32, priority int) error { - return errUnimplimented + return errUnimplemented } func (nopSyscaller) GetPriority(pid int32) (int, error) { - return 0, errUnimplimented + return 0, errUnimplemented } func (nopSyscaller) Kill(pid int32, sig syscall.Signal) error { - return errUnimplimented + return errUnimplemented } From 0ced5ce78b14da34d90a18e302a3e859618969d9 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 13 Sep 2023 00:12:28 +0000 Subject: [PATCH 13/21] skip non-linux --- agent/agentproc/proc_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 6459b05f3f7e8..1ecf27a4fe67a 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -2,6 +2,7 @@ package agentproc_test import ( "fmt" + "runtime" "strings" "syscall" "testing" @@ -18,6 +19,10 @@ import ( func TestList(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skipf("skipping non-linux environment") + } + t.Run("OK", func(t *testing.T) { t.Parallel() @@ -128,6 +133,10 @@ func TestList(t *testing.T) { func TestProcess(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skipf("skipping non-linux environment") + } + t.Run("SetOOMAdj", func(t *testing.T) { t.Parallel() From 8aaa6d569f9a8a6040c0b74650828618f9918acd Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 13 Sep 2023 01:02:06 +0000 Subject: [PATCH 14/21] skip non-linux --- agent/agent_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agent/agent_test.go b/agent/agent_test.go index c2d20209f4974..09d1e9a5fd803 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2406,6 +2406,10 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Run("OK", func(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skip("Skipping non-linux environment") + } + var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() @@ -2469,6 +2473,10 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Run("IgnoreCustomNice", func(t *testing.T) { t.Parallel() + if runtime.GOOS != "linux" { + t.Skip("Skipping non-linux environment") + } + var ( expectedProcs = map[int32]agentproc.Process{} fs = afero.NewMemMapFs() From cea4851769b71453a8cd471fe5c54ec51030d464 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 13 Sep 2023 01:45:57 +0000 Subject: [PATCH 15/21] prevent race --- agent/agent_test.go | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/agent/agent_test.go b/agent/agent_test.go index 09d1e9a5fd803..6f6f01a7d34c9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2525,14 +2525,21 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Skip("Skipping non-linux environment") } - var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) + var ( + buf bytes.Buffer + wr = &syncWriter{ + w: &buf, + } + ) + log := slog.Make(sloghuman.Sink(wr)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log }) require.Eventually(t, func() bool { + wr.mu.Lock() + defer wr.mu.Unlock() return strings.Contains(buf.String(), "process priority not enabled") }, testutil.WaitLong, testutil.IntervalFast) }) @@ -2544,8 +2551,13 @@ func TestAgent_ManageProcessPriority(t *testing.T) { t.Skip("Skipping linux environment") } - var buf bytes.Buffer - log := slog.Make(sloghuman.Sink(&buf)).Leveled(slog.LevelDebug) + var ( + buf bytes.Buffer + wr = &syncWriter{ + w: &buf, + } + ) + log := slog.Make(sloghuman.Sink(wr)).Leveled(slog.LevelDebug) _, _, _, _, _ = setupAgent(t, agentsdk.Manifest{}, 0, func(c *agenttest.Client, o *agent.Options) { o.Logger = log @@ -2554,6 +2566,9 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.EnvironmentVariables = map[string]string{agent.EnvProcPrioMgmt: "1"} }) require.Eventually(t, func() bool { + wr.mu.Lock() + defer wr.mu.Unlock() + return strings.Contains(buf.String(), "process priority not enabled") }, testutil.WaitLong, testutil.IntervalFast) }) @@ -2580,3 +2595,14 @@ func verifyCollectedMetrics(t *testing.T, expected []agentsdk.AgentMetric, actua } return true } + +type syncWriter struct { + mu sync.Mutex + w io.Writer +} + +func (s *syncWriter) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.w.Write(p) +} From 5020eb4d11e5453c841b8c63ab552cb599cc55c2 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 20:30:32 +0000 Subject: [PATCH 16/21] only prioritize ourselves --- agent/agent.go | 13 ++++++++----- agent/agent_test.go | 21 +++++++++++++-------- cli/agent.go | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 5886f305d2ca9..01fb89f25cfdf 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -79,6 +79,9 @@ type Options struct { ModifiedProcesses chan []*agentproc.Process // ProcessManagementTick is used for testing process priority management. ProcessManagementTick <-chan time.Time + // PrioritizedPIDs are processes that should have preferred CPU allocation + // and be de-prioritized for OOM Killer selection. + PrioritizedPIDs []int } type Client interface { @@ -161,6 +164,7 @@ func New(options Options) Agent { syscaller: options.Syscaller, modifiedProcs: options.ModifiedProcesses, processManagementTick: options.ProcessManagementTick, + prioritizedPIDs: options.PrioritizedPIDs, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -221,6 +225,9 @@ type agent struct { modifiedProcs chan []*agentproc.Process // processManagementTick is used for testing process priority management. processManagementTick <-chan time.Time + // prioritizedPIDs are processes that should have preferred CPU allocation + // and be de-prioritized for OOM Killer selection. + prioritizedPIDs []int } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1278,8 +1285,6 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } -var prioritizedProcs = []string{"coder"} - func (a *agent) manageProcessPriorityLoop(ctx context.Context) { if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", @@ -1337,11 +1342,9 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process slog.F("pid", proc.PID), ) - // Trim off the path e.g. "./coder" -> "coder" - name := filepath.Base(proc.Name()) // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(prioritizedProcs, name) { + if slices.Contains(a.prioritizedPIDs, int(proc.PID)) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { logger.Warn(ctx, "unable to set proc oom_score_adj", diff --git a/agent/agent_test.go b/agent/agent_test.go index 6f6f01a7d34c9..5560b5547e8c8 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2411,12 +2411,13 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var ( - expectedProcs = map[int32]agentproc.Process{} - fs = afero.NewMemMapFs() - syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) - ticker = make(chan time.Time) - modProcs = make(chan []*agentproc.Process) - logger = slog.Make(sloghuman.Sink(io.Discard)) + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + ticker = make(chan time.Time) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) + prioritizedProc = 123 ) // Create some processes. @@ -2429,12 +2430,15 @@ func TestAgent_ManageProcessPriority(t *testing.T) { proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" - p.PID = 1 + p.PID = int32(prioritizedProc) }, ) } else { // The rest are peasants. proc = agentproctest.GenerateProcess(t, fs) + if proc.PID == int32(prioritizedProc) { + proc.PID = 1234 + } syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2452,6 +2456,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker + o.PrioritizedPIDs = []int{prioritizedProc} }) actualProcs := <-modProcs require.Len(t, actualProcs, 4) @@ -2460,7 +2465,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedScore := "0" expected, ok := expectedProcs[actual.PID] require.True(t, ok) - if expected.PID == 1 { + if expected.PID == int32(prioritizedProc) { expectedScore = "-500" } diff --git a/cli/agent.go b/cli/agent.go index 6a06f4d454f91..584e40b5518be 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -299,6 +299,7 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { // Intentionally set this to nil. It's mainly used // for testing. ModifiedProcesses: nil, + PrioritizedPIDs: []int{os.Getpid()}, }) prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus") From 11ab047a6e8903c2ea72a0ab2694aa9f631c9719 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 20:57:25 +0000 Subject: [PATCH 17/21] Revert "only prioritize ourselves" This reverts commit 5020eb4d11e5453c841b8c63ab552cb599cc55c2. --- agent/agent.go | 13 +++++-------- agent/agent_test.go | 21 ++++++++------------- cli/agent.go | 1 - 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 01fb89f25cfdf..5886f305d2ca9 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -79,9 +79,6 @@ type Options struct { ModifiedProcesses chan []*agentproc.Process // ProcessManagementTick is used for testing process priority management. ProcessManagementTick <-chan time.Time - // PrioritizedPIDs are processes that should have preferred CPU allocation - // and be de-prioritized for OOM Killer selection. - PrioritizedPIDs []int } type Client interface { @@ -164,7 +161,6 @@ func New(options Options) Agent { syscaller: options.Syscaller, modifiedProcs: options.ModifiedProcesses, processManagementTick: options.ProcessManagementTick, - prioritizedPIDs: options.PrioritizedPIDs, prometheusRegistry: prometheusRegistry, metrics: newAgentMetrics(prometheusRegistry), @@ -225,9 +221,6 @@ type agent struct { modifiedProcs chan []*agentproc.Process // processManagementTick is used for testing process priority management. processManagementTick <-chan time.Time - // prioritizedPIDs are processes that should have preferred CPU allocation - // and be de-prioritized for OOM Killer selection. - prioritizedPIDs []int } func (a *agent) TailnetConn() *tailnet.Conn { @@ -1285,6 +1278,8 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } +var prioritizedProcs = []string{"coder"} + func (a *agent) manageProcessPriorityLoop(ctx context.Context) { if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", @@ -1342,9 +1337,11 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process slog.F("pid", proc.PID), ) + // Trim off the path e.g. "./coder" -> "coder" + name := filepath.Base(proc.Name()) // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(a.prioritizedPIDs, int(proc.PID)) { + if slices.Contains(prioritizedProcs, name) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { logger.Warn(ctx, "unable to set proc oom_score_adj", diff --git a/agent/agent_test.go b/agent/agent_test.go index 5560b5547e8c8..6f6f01a7d34c9 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2411,13 +2411,12 @@ func TestAgent_ManageProcessPriority(t *testing.T) { } var ( - expectedProcs = map[int32]agentproc.Process{} - fs = afero.NewMemMapFs() - syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) - ticker = make(chan time.Time) - modProcs = make(chan []*agentproc.Process) - logger = slog.Make(sloghuman.Sink(io.Discard)) - prioritizedProc = 123 + expectedProcs = map[int32]agentproc.Process{} + fs = afero.NewMemMapFs() + syscaller = agentproctest.NewMockSyscaller(gomock.NewController(t)) + ticker = make(chan time.Time) + modProcs = make(chan []*agentproc.Process) + logger = slog.Make(sloghuman.Sink(io.Discard)) ) // Create some processes. @@ -2430,15 +2429,12 @@ func TestAgent_ManageProcessPriority(t *testing.T) { proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" - p.PID = int32(prioritizedProc) + p.PID = 1 }, ) } else { // The rest are peasants. proc = agentproctest.GenerateProcess(t, fs) - if proc.PID == int32(prioritizedProc) { - proc.PID = 1234 - } syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2456,7 +2452,6 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.Filesystem = fs o.Logger = logger o.ProcessManagementTick = ticker - o.PrioritizedPIDs = []int{prioritizedProc} }) actualProcs := <-modProcs require.Len(t, actualProcs, 4) @@ -2465,7 +2460,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedScore := "0" expected, ok := expectedProcs[actual.PID] require.True(t, ok) - if expected.PID == int32(prioritizedProc) { + if expected.PID == 1 { expectedScore = "-500" } diff --git a/cli/agent.go b/cli/agent.go index 584e40b5518be..6a06f4d454f91 100644 --- a/cli/agent.go +++ b/cli/agent.go @@ -299,7 +299,6 @@ func (r *RootCmd) workspaceAgent() *clibase.Cmd { // Intentionally set this to nil. It's mainly used // for testing. ModifiedProcesses: nil, - PrioritizedPIDs: []int{os.Getpid()}, }) prometheusSrvClose := ServeHandler(ctx, logger, prometheusMetricsHandler(prometheusRegistry, logger), prometheusAddress, "prometheus") From 46ef05abef24853da4eba8155601e78d269ea47a Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 22:32:19 +0000 Subject: [PATCH 18/21] only prioritize coder agent --- agent/agent.go | 24 +++-- agent/agent_test.go | 14 ++- agent/agentproc/proc_other.go | 2 +- agent/agentproc/proc_test.go | 6 +- agent/agentproc/proc_unix.go | 10 +- agent/out.txt | 167 ++++++++++++++++++++++++++++++++++ 6 files changed, 205 insertions(+), 18 deletions(-) create mode 100644 agent/out.txt diff --git a/agent/agent.go b/agent/agent.go index 5886f305d2ca9..71272aea4121c 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -16,6 +16,7 @@ import ( "os/user" "path/filepath" "runtime" + "runtime/debug" "sort" "strconv" "strings" @@ -1278,7 +1279,7 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { } } -var prioritizedProcs = []string{"coder"} +var prioritizedProcs = []string{"coder agent"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { @@ -1290,6 +1291,15 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { return } + defer func() { + if r := recover(); r != nil { + a.logger.Critical(ctx, "recovered from panic", + slog.F("panic", r), + slog.F("stack", string(debug.Stack())), + ) + } + }() + if a.processManagementTick == nil { ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -1333,15 +1343,17 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process for _, proc := range procs { logger = a.logger.With( - slog.F("name", proc.Name()), + slog.F("cmd", proc.Cmd()), slog.F("pid", proc.PID), ) - // Trim off the path e.g. "./coder" -> "coder" - name := filepath.Base(proc.Name()) + containsFn := func(e string) bool { + contains := strings.Contains(proc.Cmd(), e) + return contains + } // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. - if slices.Contains(prioritizedProcs, name) { + if slices.ContainsFunc[[]string, string](prioritizedProcs, containsFn) { err = proc.SetOOMAdj(oomScoreAdj) if err != nil { logger.Warn(ctx, "unable to set proc oom_score_adj", @@ -1366,7 +1378,7 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process // so we don't override user nice values. // Getpriority actually returns priority for the nice value // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). - if score != 20 { + if score != 20 && score != niceness { logger.Debug(ctx, "skipping process due to custom niceness", slog.F("niceness", score), ) diff --git a/agent/agent_test.go b/agent/agent_test.go index 6f6f01a7d34c9..55c2e5411cc23 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2429,12 +2429,18 @@ func TestAgent_ManageProcessPriority(t *testing.T) { proc = agentproctest.GenerateProcess(t, fs, func(p *agentproc.Process) { p.CmdLine = "./coder\x00agent\x00--no-reap" - p.PID = 1 + p.PID = int32(i) }, ) } else { - // The rest are peasants. - proc = agentproctest.GenerateProcess(t, fs) + proc = agentproctest.GenerateProcess(t, fs, + func(p *agentproc.Process) { + // Make the cmd something similar to a prioritized + // process but differentiate the arguments. + p.CmdLine = "./coder\x00stat" + }, + ) + syscaller.EXPECT().SetPriority(proc.PID, 10).Return(nil) syscaller.EXPECT().GetPriority(proc.PID).Return(20, nil) } @@ -2460,7 +2466,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { expectedScore := "0" expected, ok := expectedProcs[actual.PID] require.True(t, ok) - if expected.PID == 1 { + if expected.PID == 0 { expectedScore = "-500" } diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go index b4d6348003572..ee1ff194103bb 100644 --- a/agent/agentproc/proc_other.go +++ b/agent/agentproc/proc_other.go @@ -19,7 +19,7 @@ func (p *Process) SetNiceness(sc Syscaller, score int) error { return errUnimplemented } -func (p *Process) Name() string { +func (p *Process) Cmd() string { return "" } diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index 1ecf27a4fe67a..c6081966904c9 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -170,16 +170,16 @@ func TestProcess(t *testing.T) { require.NoError(t, err) }) - t.Run("Name", func(t *testing.T) { + t.Run("Cmd", func(t *testing.T) { t.Parallel() var ( proc = &agentproc.Process{ CmdLine: "helloworld\x00--arg1\x00--arg2", } - expectedName = "helloworld" + expectedName = "helloworld --arg1 --arg2" ) - require.Equal(t, expectedName, proc.Name()) + require.Equal(t, expectedName, proc.Cmd()) }) } diff --git a/agent/agentproc/proc_unix.go b/agent/agentproc/proc_unix.go index 462f7b57c987c..6658c8f43a62e 100644 --- a/agent/agentproc/proc_unix.go +++ b/agent/agentproc/proc_unix.go @@ -115,8 +115,10 @@ func (p *Process) SetNiceness(sc Syscaller, score int) error { return nil } -func (p *Process) Name() string { - args := strings.Split(p.CmdLine, "\x00") - // Split will always return at least one element. - return args[0] +func (p *Process) Cmd() string { + return strings.Join(p.cmdLine(), " ") +} + +func (p *Process) cmdLine() []string { + return strings.Split(p.CmdLine, "\x00") } diff --git a/agent/out.txt b/agent/out.txt new file mode 100644 index 0000000000000..7b435abf43cb0 --- /dev/null +++ b/agent/out.txt @@ -0,0 +1,167 @@ +=== RUN TestAgent_ManageProcessPriority +=== PAUSE TestAgent_ManageProcessPriority +=== CONT TestAgent_ManageProcessPriority +=== RUN TestAgent_ManageProcessPriority/OK +=== PAUSE TestAgent_ManageProcessPriority/OK +=== CONT TestAgent_ManageProcessPriority/OK +YEAH +YUP + t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: get service banner + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) tun device + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) OS network configurator + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) DNS configurator + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: dns: using dns.noopManager + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: link state: interfaces.State{defaultRoute=eth0 ifs={docker0:[172.17.0.1/16] eth0:[172.20.0.7/16]} v4=true v6=false} + t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v4 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v6 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: disco key = d:9bf350ad1946674e + t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: Creating WireGuard device... + t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post lifecycle req={"state":"starting","changed_at":"2023-09-14T22:21:04.162755Z"} + t.go:85: 2023-09-14 22:21:04.163 [debu] agent.client: post lifecycle req={"state":"ready","changed_at":"2023-09-14T22:21:04.162813Z"} + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing WireGuard device up... + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] UDP bind has been updated + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Interface state was Down, requested Up, now Up + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing router up... + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: [v1] warning: fakeRouter.Up: not implemented. + t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: Clearing router settings... + t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - started + t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. + t.go:85: 2023-09-14 22:21:04.165 [debu] client.net.wgengine: Starting network monitor... + t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: Engine created. + t.go:85: 2023-09-14 22:21:04.166 [debu] client: magicsock debug logging disabled, use CODER_MAGICSOCK_DEBUG_LOGGING=true to enable + t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: magicsock: SetPrivateKey called (init) + t.go:85: 2023-09-14 22:21:04.170 [debu] client: updating network map + t.go:85: 2023-09-14 22:21:04.170 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 0 peers + t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating client client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.170 [debu] client: skipped sending node; no PreferredDERP node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.170813 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:0 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" + t.go:85: 2023-09-14 22:21:04.170 [debu] multiagent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.171 [debu] agent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.171 [debu] wrote initial client(s) to agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] + t.go:85: 2023-09-14 22:21:04.171 [debu] added agent socket agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer + t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer + t.go:85: 2023-09-14 22:21:04.171 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] + t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (6f93c36c4883ce76647c9e74) in 4.705998ms + t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (3d5303cc01a8f9a035b236a4) in 4.76389ms + t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (a41d11ea5ec51a69012804ae) in 4.792004ms + t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer + t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: home is now derp-1 (test) + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: adding connection to derp-1 for home-keep-alive + t.go:85: 2023-09-14 22:21:04.228 [debu] client: netinfo callback netinfo="NetInfo{varies=false hairpin= ipv6=false ipv6os=false udp=true icmpv4=false derp=#1 portmap= link=\"\"}" + t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228454 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: 1 active derp conns: derp-1=cr0s,wr0s + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: endpoints changed: 127.0.0.1:39073 (stun), 172.17.0.1:39073 (local), 172.20.0.7:39073 (local) + t.go:85: 2023-09-14 22:21:04.228 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228454Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":[]} + t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] + t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228713945 +0000 UTC m=+0.212205983 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= + t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: derphttp.Client.Connect: connecting to derp-1 (test) + t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228760465 +0000 UTC m=+0.212252503 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= + t.go:85: 2023-09-14 22:21:04.228 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.228767Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":[]} + t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228869 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[127.0.0.1:39073 172.17.0.1:39073 172.20.0.7:39073]}" + t.go:85: 2023-09-14 22:21:04.228 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228454Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[]}]" + t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 + t.go:85: 2023-09-14 22:21:04.229 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228869Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":["127.0.0.1:39073","172.17.0.1:39073","172.20.0.7:39073"]} + t.go:85: 2023-09-14 22:21:04.229 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] + t.go:85: 2023-09-14 22:21:04.229 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.228767 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[]}" + t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.228767Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[]}]" + t.go:85: 2023-09-14 22:21:04.229 [debu] client: updating network map + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.netstack: [v2] netstack: registered IP fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128 + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring userspace WireGuard config (with 1/1 peers) + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] UAPI: Updating private key + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Created + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (d1abaaf30fcfbb0a2e30af00) in 68.382µs + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating endpoint + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (bbe130a507ad56b3f11e5620) in 104.803µs + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Removing all allowedips + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (f3e4b4d721a0986d001d0c52) in 129.58µs + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating persistent keepalive interval + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Starting + t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228869Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[\"127.0.0.1:39073\",\"172.17.0.1:39073\",\"172.20.0.7:39073\"]}]" + t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring router + t.go:85: 2023-09-14 22:21:04.229 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.229132Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":["127.0.0.1:32901","172.17.0.1:32901","172.20.0.7:32901"]} + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: wgengine: Reconfig: configuring DNS + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Set: {DefaultResolvers:[] Routes:{} SearchDomains:[] Hosts:0} + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Resolvercfg: {Routes:{} Hosts:0 LocalDomains:[]} + t.go:85: 2023-09-14 22:21:04.230 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: OScfg: {Nameservers:[] SearchDomains:[] MatchDomains:[] Hosts:[]} + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done + t.go:85: 2023-09-14 22:21:04.230 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.229132Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[\"127.0.0.1:32901\",\"172.17.0.1:32901\",\"172.20.0.7:32901\"]}]" + t.go:85: 2023-09-14 22:21:04.230 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.229132 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[127.0.0.1:32901 172.17.0.1:32901 172.20.0.7:32901]}" + t.go:85: 2023-09-14 22:21:04.230 [debu] client: updating network map + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers + t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done + t.go:85: 2023-09-14 22:21:04.232 [debu] client.net.wgengine: magicsock: derp-1 connected; connGen=1 + t.go:85: 2023-09-14 22:21:04.280 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted + t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) + t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 + t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): sending disco ping to [Vs9bR] ... + t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: [v1] magicsock: derp route for [Vs9bR] set to derp-1 (shared home) + t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 172.17.0.1:32901 + t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 127.0.0.1:32901 + agent_test.go:2463: + Error Trace: /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2463 + Error: "[]" should have 4 item(s), but has 0 + Test: TestAgent_ManageProcessPriority/OK + t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using DERP only (reset) + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: closing connection to derp-1 (conn-close), age 0s + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: 0 active derp conns + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: [v1] warning: fakeRouter.Close: not implemented. + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closing + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - stopped + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Stopping + t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closed + t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read agent update, connection may be closed agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... + error= read json: + github.com/coder/coder/v2/tailnet.(*coordinator).handleNextAgentMessage + /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:552 + - io: read/write on closed pipe + t.go:85: 2023-09-14 22:21:04.298 [debu] deleted agent socket and node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.298 [debu] done sending updates agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read client update, connection may be closed client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... + error= read json: + github.com/coder/coder/v2/tailnet.(*coordinator).handleNextClientMessage + /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:350 + - EOF + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client node client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted last client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 + t.go:85: 2023-09-14 22:21:04.299 [debu] done sending updates client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a + t.go:85: 2023-09-14 22:21:04.299 [debu] agent.client: post lifecycle req={"state":"shutting_down","changed_at":"2023-09-14T22:21:04.299336Z"} + t.go:85: 2023-09-14 22:21:04.499 [debu] agent.client: get service banner + t.go:85: 2023-09-14 22:21:04.500 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} + t.go:85: 2023-09-14 22:21:06.300 [debu] agent.client: post lifecycle req={"state":"off","changed_at":"2023-09-14T22:21:04.299369Z"} + stuntest.go:63: STUN server shutdown + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 0 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1965978598 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1248996486 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1788987366 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1965978598 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1248996486 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1788987366 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1965978598 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1248996486 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 + controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1788987366 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 + controller.go:137: aborting test due to missing call(s) +--- FAIL: TestAgent_ManageProcessPriority (0.00s) + --- FAIL: TestAgent_ManageProcessPriority/OK (2.26s) +FAIL +exit status 1 +FAIL github.com/coder/coder/v2/agent 2.294s From d132480a017450421d2ea0e6357e6c444b914996 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 14 Sep 2023 23:12:43 +0000 Subject: [PATCH 19/21] remove oom_score_adj --- agent/agent.go | 13 +- agent/agent_test.go | 15 +-- agent/agentproc/agentproctest/proc.go | 4 - agent/agentproc/proc_other.go | 4 - agent/agentproc/proc_test.go | 19 --- agent/agentproc/proc_unix.go | 15 --- agent/agentproc/syscaller.go | 3 - agent/out.txt | 167 -------------------------- 8 files changed, 3 insertions(+), 237 deletions(-) delete mode 100644 agent/out.txt diff --git a/agent/agent.go b/agent/agent.go index 71272aea4121c..9babab81d8475 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1327,8 +1327,7 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process, error) { const ( - niceness = 10 - oomScoreAdj = -500 + niceness = 10 ) procs, err := agentproc.List(a.filesystem, a.syscaller) @@ -1351,18 +1350,10 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process contains := strings.Contains(proc.Cmd(), e) return contains } + // If the process is prioritized we should adjust // it's oom_score_adj and avoid lowering its niceness. if slices.ContainsFunc[[]string, string](prioritizedProcs, containsFn) { - err = proc.SetOOMAdj(oomScoreAdj) - if err != nil { - logger.Warn(ctx, "unable to set proc oom_score_adj", - slog.F("oom_score_adj", oomScoreAdj), - slog.Error(err), - ) - continue - } - modProcs = append(modProcs, proc) continue } diff --git a/agent/agent_test.go b/agent/agent_test.go index 55c2e5411cc23..80fa7435c7437 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -2460,20 +2460,7 @@ func TestAgent_ManageProcessPriority(t *testing.T) { o.ProcessManagementTick = ticker }) actualProcs := <-modProcs - require.Len(t, actualProcs, 4) - - for _, actual := range actualProcs { - expectedScore := "0" - expected, ok := expectedProcs[actual.PID] - require.True(t, ok) - if expected.PID == 0 { - expectedScore = "-500" - } - - score, err := afero.ReadFile(fs, filepath.Join(actual.Dir, "oom_score_adj")) - require.NoError(t, err) - require.Equal(t, expectedScore, strings.TrimSpace(string(score))) - } + require.Len(t, actualProcs, len(expectedProcs)-1) }) t.Run("IgnoreCustomNice", func(t *testing.T) { diff --git a/agent/agentproc/agentproctest/proc.go b/agent/agentproc/agentproctest/proc.go index 1fe44ad9eb31b..c36e04ec1cdc3 100644 --- a/agent/agentproc/agentproctest/proc.go +++ b/agent/agentproc/agentproctest/proc.go @@ -31,7 +31,6 @@ func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process) process := agentproc.Process{ CmdLine: cmdline, PID: int32(pid), - FS: fs, } for _, mut := range muts { @@ -46,8 +45,5 @@ func GenerateProcess(t *testing.T, fs afero.Fs, muts ...func(*agentproc.Process) err = afero.WriteFile(fs, fmt.Sprintf("%s/cmdline", process.Dir), []byte(process.CmdLine), 0o444) require.NoError(t, err) - err = afero.WriteFile(fs, fmt.Sprintf("%s/oom_score_adj", process.Dir), []byte("0"), 0o444) - require.NoError(t, err) - return process } diff --git a/agent/agentproc/proc_other.go b/agent/agentproc/proc_other.go index ee1ff194103bb..c0c4e2a25ce32 100644 --- a/agent/agentproc/proc_other.go +++ b/agent/agentproc/proc_other.go @@ -7,10 +7,6 @@ import ( "github.com/spf13/afero" ) -func (p *Process) SetOOMAdj(score int) error { - return errUnimplemented -} - func (p *Process) Niceness(sc Syscaller) (int, error) { return 0, errUnimplemented } diff --git a/agent/agentproc/proc_test.go b/agent/agentproc/proc_test.go index c6081966904c9..37991679503c6 100644 --- a/agent/agentproc/proc_test.go +++ b/agent/agentproc/proc_test.go @@ -1,9 +1,7 @@ package agentproc_test import ( - "fmt" "runtime" - "strings" "syscall" "testing" @@ -137,23 +135,6 @@ func TestProcess(t *testing.T) { t.Skipf("skipping non-linux environment") } - t.Run("SetOOMAdj", func(t *testing.T) { - t.Parallel() - - var ( - fs = afero.NewMemMapFs() - proc = agentproctest.GenerateProcess(t, fs) - expectedScore = -1000 - ) - - err := proc.SetOOMAdj(expectedScore) - require.NoError(t, err) - - actualScore, err := afero.ReadFile(fs, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID)) - require.NoError(t, err) - require.Equal(t, fmt.Sprintf("%d", expectedScore), strings.TrimSpace(string(actualScore))) - }) - t.Run("SetNiceness", func(t *testing.T) { t.Parallel() diff --git a/agent/agentproc/proc_unix.go b/agent/agentproc/proc_unix.go index 6658c8f43a62e..f52caed52ee33 100644 --- a/agent/agentproc/proc_unix.go +++ b/agent/agentproc/proc_unix.go @@ -54,7 +54,6 @@ func List(fs afero.Fs, syscaller Syscaller) ([]*Process, error) { PID: int32(pid), CmdLine: string(cmdline), Dir: filepath.Join(defaultProcDir, entry), - FS: fs, }) } @@ -85,20 +84,6 @@ func isProcessExist(syscaller Syscaller, pid int32) (bool, error) { return false, xerrors.Errorf("kill: %w", err) } -func (p *Process) SetOOMAdj(score int) error { - path := filepath.Join(p.Dir, "oom_score_adj") - err := afero.WriteFile(p.FS, - path, - []byte(strconv.Itoa(score)), - 0o644, - ) - if err != nil { - return xerrors.Errorf("write %q: %w", path, err) - } - - return nil -} - func (p *Process) Niceness(sc Syscaller) (int, error) { nice, err := sc.GetPriority(p.PID) if err != nil { diff --git a/agent/agentproc/syscaller.go b/agent/agentproc/syscaller.go index cf2c90d7ec77d..1cd6640e36b43 100644 --- a/agent/agentproc/syscaller.go +++ b/agent/agentproc/syscaller.go @@ -2,8 +2,6 @@ package agentproc import ( "syscall" - - "github.com/spf13/afero" ) type Syscaller interface { @@ -18,5 +16,4 @@ type Process struct { Dir string CmdLine string PID int32 - FS afero.Fs } diff --git a/agent/out.txt b/agent/out.txt deleted file mode 100644 index 7b435abf43cb0..0000000000000 --- a/agent/out.txt +++ /dev/null @@ -1,167 +0,0 @@ -=== RUN TestAgent_ManageProcessPriority -=== PAUSE TestAgent_ManageProcessPriority -=== CONT TestAgent_ManageProcessPriority -=== RUN TestAgent_ManageProcessPriority/OK -=== PAUSE TestAgent_ManageProcessPriority/OK -=== CONT TestAgent_ManageProcessPriority/OK -YEAH -YUP - t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: get service banner - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) tun device - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) OS network configurator - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] using fake (no-op) DNS configurator - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: dns: using dns.noopManager - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: link state: interfaces.State{defaultRoute=eth0 ifs={docker0:[172.17.0.1/16] eth0:[172.20.0.7/16]} v4=true v6=false} - t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP read buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: [warning] failed to force-set UDP write buffer size to 7340032: operation not permitted; using kernel default values (impacts throughput only) - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v4 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: [v1] couldn't create raw v6 disco listener, using regular listener instead: raw disco listening disabled, SO_MARK unavailable - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: magicsock: disco key = d:9bf350ad1946674e - t.go:85: 2023-09-14 22:21:04.162 [debu] client.net.wgengine: Creating WireGuard device... - t.go:85: 2023-09-14 22:21:04.162 [debu] agent.client: post lifecycle req={"state":"starting","changed_at":"2023-09-14T22:21:04.162755Z"} - t.go:85: 2023-09-14 22:21:04.163 [debu] agent.client: post lifecycle req={"state":"ready","changed_at":"2023-09-14T22:21:04.162813Z"} - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing WireGuard device up... - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] UDP bind has been updated - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Interface state was Down, requested Up, now Up - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: Bringing router up... - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - started - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: [v1] warning: fakeRouter.Up: not implemented. - t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: Clearing router settings... - t.go:85: 2023-09-14 22:21:04.163 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - started - t.go:85: 2023-09-14 22:21:04.164 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. - t.go:85: 2023-09-14 22:21:04.165 [debu] client.net.wgengine: Starting network monitor... - t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: Engine created. - t.go:85: 2023-09-14 22:21:04.166 [debu] client: magicsock debug logging disabled, use CODER_MAGICSOCK_DEBUG_LOGGING=true to enable - t.go:85: 2023-09-14 22:21:04.166 [debu] client.net.wgengine: magicsock: SetPrivateKey called (init) - t.go:85: 2023-09-14 22:21:04.170 [debu] client: updating network map - t.go:85: 2023-09-14 22:21:04.170 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 0 peers - t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating client client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.170 [debu] client: skipped sending node; no PreferredDERP node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.170813 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:0 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" - t.go:85: 2023-09-14 22:21:04.170 [debu] multiagent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.170 [debu] coordinating agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.171 [debu] agent node doesn't exist client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.171 [debu] wrote initial client(s) to agent agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] - t.go:85: 2023-09-14 22:21:04.171 [debu] added agent socket agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer - t.go:85: 2023-09-14 22:21:04.171 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer - t.go:85: 2023-09-14 22:21:04.171 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes=[] - t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (6f93c36c4883ce76647c9e74) in 4.705998ms - t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (3d5303cc01a8f9a035b236a4) in 4.76389ms - t.go:85: 2023-09-14 22:21:04.175 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (a41d11ea5ec51a69012804ae) in 4.792004ms - t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): no matching peer - t.go:85: 2023-09-14 22:21:04.221 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: home is now derp-1 (test) - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: adding connection to derp-1 for home-keep-alive - t.go:85: 2023-09-14 22:21:04.228 [debu] client: netinfo callback netinfo="NetInfo{varies=false hairpin= ipv6=false ipv6os=false udp=true icmpv4=false derp=#1 portmap= link=\"\"}" - t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228454 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[]}" - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: 1 active derp conns: derp-1=cr0s,wr0s - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: magicsock: endpoints changed: 127.0.0.1:39073 (stun), 172.17.0.1:39073 (local), 172.20.0.7:39073 (local) - t.go:85: 2023-09-14 22:21:04.228 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228454Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":[]} - t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] - t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228713945 +0000 UTC m=+0.212205983 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= - t.go:85: 2023-09-14 22:21:04.228 [debu] client.net.wgengine: derphttp.Client.Connect: connecting to derp-1 (test) - t.go:85: 2023-09-14 22:21:04.228 [debu] client: wireguard status status="&{AsOf:2023-09-14 22:21:04.228760465 +0000 UTC m=+0.212252503 Peers:[] LocalAddrs:[{Addr:127.0.0.1:39073 Type:stun} {Addr:172.17.0.1:39073 Type:local} {Addr:172.20.0.7:39073 Type:local}] DERPs:1}" error= - t.go:85: 2023-09-14 22:21:04.228 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.228767Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":[]} - t.go:85: 2023-09-14 22:21:04.228 [debu] client: sending node node="&{ID:nodeid:1335e86dcc92b903 AsOf:2023-09-14 22:21:04.228869 +0000 UTC Key:nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24 DiscoKey:discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] AllowedIPs:[fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128] Endpoints:[127.0.0.1:39073 172.17.0.1:39073 172.20.0.7:39073]}" - t.go:85: 2023-09-14 22:21:04.228 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228454Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[]}]" - t.go:85: 2023-09-14 22:21:04.228 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 - t.go:85: 2023-09-14 22:21:04.229 [debu] got client node update client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 node={"id":1384268018767542531,"as_of":"2023-09-14T22:21:04.228869Z","key":"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24","disco":"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"allowed_ips":["fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128"],"endpoints":["127.0.0.1:39073","172.17.0.1:39073","172.20.0.7:39073"]} - t.go:85: 2023-09-14 22:21:04.229 [debu] enqueued node to agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_ids=[4db2658f-194a-4b75-9608-1b6b42d6ba8a] - t.go:85: 2023-09-14 22:21:04.229 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.228767 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[]}" - t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.228767Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[]}]" - t.go:85: 2023-09-14 22:21:04.229 [debu] client: updating network map - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.netstack: [v2] netstack: registered IP fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128 - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring userspace WireGuard config (with 1/1 peers) - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] UAPI: Updating private key - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Created - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (d1abaaf30fcfbb0a2e30af00) in 68.382µs - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating endpoint - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (bbe130a507ad56b3f11e5620) in 104.803µs - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Removing all allowedips - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: netcheck: netcheck.runProbe: got STUN response for t2 from 127.0.0.1:39073 (f3e4b4d721a0986d001d0c52) in 129.58µs - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Adding allowedip - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - UAPI: Updating persistent keepalive interval - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Starting - t.go:85: 2023-09-14 22:21:04.229 [debu] wrote nodes agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":1384268018767542531,\"as_of\":\"2023-09-14T22:21:04.228869Z\",\"key\":\"nodekey:ab1def7f7652884e18276c7691dbdb40e18ce0adfdcd02f604eb28ea6c0f2b24\",\"disco\":\"discokey:9bf350ad1946674ea63bdef15c860c8669b5cf5a092090de5950676bf3a99a10\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4320:834e:19a:f9e:1b7a/128\"],\"endpoints\":[\"127.0.0.1:39073\",\"172.17.0.1:39073\",\"172.20.0.7:39073\"]}]" - t.go:85: 2023-09-14 22:21:04.229 [debu] client.net.wgengine: wgengine: Reconfig: configuring router - t.go:85: 2023-09-14 22:21:04.229 [debu] decoded agent node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a node={"id":7635823020709856630,"as_of":"2023-09-14T22:21:04.229132Z","key":"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d","disco":"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352","preferred_derp":1,"derp_latency":{},"derp_forced_websockets":{},"addresses":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"allowed_ips":["fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128","fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128"],"endpoints":["127.0.0.1:32901","172.17.0.1:32901","172.20.0.7:32901"]} - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] warning: fakeRouter.Set: not implemented. - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: wgengine: Reconfig: configuring DNS - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Set: {DefaultResolvers:[] Routes:{} SearchDomains:[] Hosts:0} - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: Resolvercfg: {Routes:{} Hosts:0 LocalDomains:[]} - t.go:85: 2023-09-14 22:21:04.230 [debu] enqueued agent node to client agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: dns: OScfg: {Nameservers:[] SearchDomains:[] MatchDomains:[] Hosts:[]} - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done - t.go:85: 2023-09-14 22:21:04.230 [debu] wrote nodes client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a nodes="[{\"id\":7635823020709856630,\"as_of\":\"2023-09-14T22:21:04.229132Z\",\"key\":\"nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d\",\"disco\":\"discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352\",\"preferred_derp\":1,\"derp_latency\":{},\"derp_forced_websockets\":{},\"addresses\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"allowed_ips\":[\"fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128\",\"fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128\"],\"endpoints\":[\"127.0.0.1:32901\",\"172.17.0.1:32901\",\"172.20.0.7:32901\"]}]" - t.go:85: 2023-09-14 22:21:04.230 [debu] client: adding node node="&{ID:nodeid:69f7e494bd294576 AsOf:2023-09-14 22:21:04.229132 +0000 UTC Key:nodekey:56cf5b45c3303aca37825498cc3ec6f30830d6485b3fc04c3a29c36ebdfc1c5d DiscoKey:discokey:d758cdfe4a6adf9fce991c496991ad7d2ef61111f148e07f2d160ea6b9bb9352 PreferredDERP:1 DERPLatency:map[] DERPForcedWebsocket:map[] Addresses:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] AllowedIPs:[fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a/128 fd7a:115c:a1e0:49d6:b259:b7ac:b1b2:48f4/128] Endpoints:[127.0.0.1:32901 172.17.0.1:32901 172.20.0.7:32901]}" - t.go:85: 2023-09-14 22:21:04.230 [debu] client: updating network map - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] magicsock: got updated network map; 1 peers - t.go:85: 2023-09-14 22:21:04.230 [debu] client.net.wgengine: [v1] wgengine: Reconfig done - t.go:85: 2023-09-14 22:21:04.232 [debu] client.net.wgengine: magicsock: derp-1 connected; connGen=1 - t.go:85: 2023-09-14 22:21:04.280 [debu] client.net.wgengine: netcheck: [v1] measuring ICMP latency of test (1): listen ip4:icmp 0.0.0.0: socket: operation not permitted - t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] netcheck: measuring HTTP(S) latency of test (1): unexpected status code: 426 (426 Upgrade Required) - t.go:85: 2023-09-14 22:21:04.286 [debu] client.net.wgengine: netcheck: [v1] report: udp=true v6=false v6os=false mapvarydest=false hair= portmap= v4a=127.0.0.1:39073 derp=0 - t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: ping(fd7a:115c:a1e0:4b75:9608:1b6b:42d6:ba8a): sending disco ping to [Vs9bR] ... - t.go:85: 2023-09-14 22:21:04.296 [debu] client.net.wgengine: [v1] magicsock: derp route for [Vs9bR] set to derp-1 (shared home) - t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 172.17.0.1:32901 - t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using 127.0.0.1:32901 - agent_test.go:2463: - Error Trace: /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2463 - Error: "[]" should have 4 item(s), but has 0 - Test: TestAgent_ManageProcessPriority/OK - t.go:85: 2023-09-14 22:21:04.297 [debu] client.net.wgengine: magicsock: disco: node [Vs9bR] d:d758cdfe4a6adf9f now using DERP only (reset) - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: closing connection to derp-1 (conn-close), age 0s - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: magicsock: 0 active derp conns - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: [v1] warning: fakeRouter.Close: not implemented. - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closing - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming mkReceiveFunc - stopped - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Routine: receive incoming receiveDERP - stopped - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] [Vs9bR] - Stopping - t.go:85: 2023-09-14 22:21:04.298 [debu] client.net.wgengine: wg: [v2] Device closed - t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read agent update, connection may be closed agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... - error= read json: - github.com/coder/coder/v2/tailnet.(*coordinator).handleNextAgentMessage - /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:552 - - io: read/write on closed pipe - t.go:85: 2023-09-14 22:21:04.298 [debu] deleted agent socket and node agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.298 [debu] done sending updates agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.298 [debu] unable to read client update, connection may be closed client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a ... - error= read json: - github.com/coder/coder/v2/tailnet.(*coordinator).handleNextClientMessage - /home/coder/go/src/github.com/coder/coder/tailnet/coordinator.go:350 - - EOF - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client node client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted last client connectionSocket from map client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.299 [debu] deleted client agents client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=00000000-0000-0000-0000-000000000000 - t.go:85: 2023-09-14 22:21:04.299 [debu] done sending updates client_id=af26bfd4-4805-4d56-961c-cd18a1dc0214 agent_id=4db2658f-194a-4b75-9608-1b6b42d6ba8a - t.go:85: 2023-09-14 22:21:04.299 [debu] agent.client: post lifecycle req={"state":"shutting_down","changed_at":"2023-09-14T22:21:04.299336Z"} - t.go:85: 2023-09-14 22:21:04.499 [debu] agent.client: get service banner - t.go:85: 2023-09-14 22:21:04.500 [debu] agent.client: post startup req={"version":"v0.0.0-devel","expanded_directory":"","subsystems":null} - t.go:85: 2023-09-14 22:21:06.300 [debu] agent.client: post lifecycle req={"state":"off","changed_at":"2023-09-14T22:21:04.299369Z"} - stuntest.go:63: STUN server shutdown - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 0 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1965978598 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1248996486 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.Kill(is equal to 1788987366 (int32), is equal to signal 0 (syscall.Signal)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2448 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1965978598 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1248996486 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.SetPriority(is equal to 1788987366 (int32), is equal to 10 (int)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2444 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1965978598 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1248996486 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 - controller.go:137: missing call(s) to *agentproctest.MockSyscaller.GetPriority(is equal to 1788987366 (int32)) /home/coder/go/src/github.com/coder/coder/agent/agent_test.go:2445 - controller.go:137: aborting test due to missing call(s) ---- FAIL: TestAgent_ManageProcessPriority (0.00s) - --- FAIL: TestAgent_ManageProcessPriority/OK (2.26s) -FAIL -exit status 1 -FAIL github.com/coder/coder/v2/agent 2.294s From 04ee5cb5b27e4b3cf2e904931fccac8e248f513d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 15 Sep 2023 00:18:34 +0000 Subject: [PATCH 20/21] defer first --- agent/agent.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index 9babab81d8475..d39543324539f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1282,15 +1282,6 @@ func (a *agent) startReportingConnectionStats(ctx context.Context) { var prioritizedProcs = []string{"coder agent"} func (a *agent) manageProcessPriorityLoop(ctx context.Context) { - if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { - a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", - slog.F("env_var", EnvProcPrioMgmt), - slog.F("value", val), - slog.F("goos", runtime.GOOS), - ) - return - } - defer func() { if r := recover(); r != nil { a.logger.Critical(ctx, "recovered from panic", @@ -1300,6 +1291,15 @@ func (a *agent) manageProcessPriorityLoop(ctx context.Context) { } }() + if val := a.envVars[EnvProcPrioMgmt]; val == "" || runtime.GOOS != "linux" { + a.logger.Debug(ctx, "process priority not enabled, agent will not manage process niceness/oom_score_adj ", + slog.F("env_var", EnvProcPrioMgmt), + slog.F("value", val), + slog.F("goos", runtime.GOOS), + ) + return + } + if a.processManagementTick == nil { ticker := time.NewTicker(time.Second) defer ticker.Stop() From ffbeab963fbd392f2e3701eec37382c6e3b8affb Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Fri, 15 Sep 2023 00:21:52 +0000 Subject: [PATCH 21/21] avoid resetting niceness for already niced procs --- agent/agent.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/agent/agent.go b/agent/agent.go index d39543324539f..cdab56d935ce3 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1369,10 +1369,12 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process // so we don't override user nice values. // Getpriority actually returns priority for the nice value // which is niceness + 20, so here 20 = a niceness of 0 (aka unset). - if score != 20 && score != niceness { - logger.Debug(ctx, "skipping process due to custom niceness", - slog.F("niceness", score), - ) + if score != 20 { + if score != niceness { + logger.Debug(ctx, "skipping process due to custom niceness", + slog.F("niceness", score), + ) + } continue }