Skip to content

feat: support adjusting child proc oom scores #12655

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Apr 3, 2024
Merged

feat: support adjusting child proc oom scores #12655

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
02eb747
fix: adjust process oom_score_adj to 0
sreya Mar 19, 2024
b3a787a
fmt
sreya Mar 19, 2024
f37242b
oom score
sreya Mar 20, 2024
403a637
fmt
sreya Mar 20, 2024
a214974
lint
sreya Mar 21, 2024
54b7f4c
Merge branch 'main' into jon/procprio
sreya Apr 2, 2024
17714ed
debounce logs
sreya Apr 2, 2024
561279b
lint
sreya Apr 2, 2024
2ddeb8e
fmt
sreya Apr 2, 2024
9f5f63c
pass through env var
sreya Apr 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
fix: adjust process oom_score_adj to 0
  • Loading branch information
@sreya
sreya committed Mar 19, 2024
commit 02eb7476561b440850b8566bd0199f32403fa687
29 changes: 26 additions & 3 deletions agent/agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1569,6 +1569,20 @@ func (a *agent) manageProcessPriorityUntilGracefulShutdown() {
return
}

const agentOOMScore = "-1000"

err := afero.WriteFile(a.filesystem, "/proc/self/oom_score_adj", []byte(agentOOMScore), 0600)
if err != nil {
a.logger.Error(ctx, "error adjusting agent oom_score_adj",
slog.F("score", agentOOMScore),
slog.Error(err),
)
} else {
a.logger.Debug(ctx, "adjusted agent oom_score_adj to avoid OOM Killer",
slog.F("score", agentOOMScore),
)
}

if a.processManagementTick == nil {
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
Expand Down Expand Up @@ -1622,12 +1636,12 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process

// If the process is prioritized we should adjust
// it's oom_score_adj and avoid lowering its niceness.
if slices.ContainsFunc[[]string, string](prioritizedProcs, containsFn) {
if slices.ContainsFunc(prioritizedProcs, containsFn) {
continue
}

score, err := proc.Niceness(a.syscaller)
if err != nil {
if err != nil && !xerrors.Is(err, os.ErrPermission) {
logger.Warn(ctx, "unable to get proc niceness",
slog.Error(err),
)
Expand All @@ -1644,14 +1658,23 @@ func (a *agent) manageProcessPriority(ctx context.Context) ([]*agentproc.Process
}

err = proc.SetNiceness(a.syscaller, niceness)
if err != nil {
if err != nil && !xerrors.Is(err, os.ErrPermission) {
logger.Warn(ctx, "unable to set proc niceness",
slog.F("niceness", niceness),
slog.Error(err),
)
continue
}

err = afero.WriteFile(a.filesystem, fmt.Sprintf("/proc/%d/oom_score_adj", proc.PID), []byte("0"), 0600)
if err != nil && !xerrors.Is(err, os.ErrPermission) {
logger.Warn(ctx, "unable to set oom_score_adj",
slog.F("score", "0"),
slog.Error(err),
)
continue
}

modProcs = append(modProcs, proc)
}
return modProcs, nil
Expand Down
15 changes: 13 additions & 2 deletions agent/agent_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2589,8 +2589,16 @@ func TestAgent_ManageProcessPriority(t *testing.T) {
logger = slog.Make(sloghuman.Sink(io.Discard))
)

requireScore := func(t *testing.T, p *agentproc.Process, score string) {
t.Helper()

actual, err := afero.ReadFile(fs, fmt.Sprintf("/proc/%d/oom_score_adj", p.PID))
require.NoError(t, err)
require.Equal(t, score, string(actual))
}

// Create some processes.
for i := 0; i < 2; i++ {
for i := 0; i < 3; i++ {
proc := agentproctest.GenerateProcess(t, fs)
syscaller.EXPECT().
Kill(proc.PID, syscall.Signal(0)).
Expand Down Expand Up @@ -2618,7 +2626,10 @@ func TestAgent_ManageProcessPriority(t *testing.T) {
})
actualProcs := <-modProcs
// We should ignore the process with a custom nice score.
require.Len(t, actualProcs, 1)
require.Len(t, actualProcs, 2)
for _, proc := range actualProcs {
requireScore(t, proc, "0")
}
})

t.Run("DisabledByDefault", func(t *testing.T) {
Expand Down