Skip to content

Commit 13b89f7

Browse files
mtojekmtojek
authored
feat: purge old provisioner daemons (coder#10949)
1 parent 153abd5 commit 13b89f7

File tree

9 files changed

+198
-27
lines changed

9 files changed

+198
-27
lines changed

‎coderd/database/dbauthz/dbauthz.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -791,6 +791,13 @@ func (q *querier) DeleteLicense(ctx context.Context, id int32) (int32, error) {
791791
return id, nil
792792
}
793793

794+
func (q *querier) DeleteOldProvisionerDaemons(ctx context.Context) error {
795+
if err := q.authorizeContext(ctx, rbac.ActionDelete, rbac.ResourceSystem); err != nil {
796+
return err
797+
}
798+
return q.db.DeleteOldProvisionerDaemons(ctx)
799+
}
800+
794801
func (q *querier) DeleteOldWorkspaceAgentLogs(ctx context.Context) error {
795802
if err := q.authorizeContext(ctx, rbac.ActionDelete, rbac.ResourceSystem); err != nil {
796803
return err

‎coderd/database/dbmem/dbmem.go

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1112,8 +1112,27 @@ func (q *FakeQuerier) DeleteLicense(_ context.Context, id int32) (int32, error)
11121112
return 0, sql.ErrNoRows
11131113
}
11141114

1115+
func (q *FakeQuerier) DeleteOldProvisionerDaemons(_ context.Context) error {
1116+
q.mutex.Lock()
1117+
defer q.mutex.Unlock()
1118+
1119+
now := dbtime.Now()
1120+
weekInterval := 7 * 24 * time.Hour
1121+
weekAgo := now.Add(-weekInterval)
1122+
1123+
var validDaemons []database.ProvisionerDaemon
1124+
for _, p := range q.provisionerDaemons {
1125+
if (p.CreatedAt.Before(weekAgo) && !p.UpdatedAt.Valid) || (p.UpdatedAt.Valid && p.UpdatedAt.Time.Before(weekAgo)) {
1126+
continue
1127+
}
1128+
validDaemons = append(validDaemons, p)
1129+
}
1130+
q.provisionerDaemons = validDaemons
1131+
return nil
1132+
}
1133+
11151134
func (*FakeQuerier) DeleteOldWorkspaceAgentLogs(_ context.Context) error {
1116-
// noop
1135+
// no-op
11171136
return nil
11181137
}
11191138

@@ -4845,6 +4864,7 @@ func (q *FakeQuerier) InsertProvisionerDaemon(_ context.Context, arg database.In
48454864
Name: arg.Name,
48464865
Provisioners: arg.Provisioners,
48474866
Tags: arg.Tags,
4867+
UpdatedAt: arg.UpdatedAt,
48484868
}
48494869
q.provisionerDaemons = append(q.provisionerDaemons, daemon)
48504870
return daemon, nil

‎coderd/database/dbmetrics/dbmetrics.go

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbmock/dbmock.go

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbpurge/dbpurge.go

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@ import (
99
"golang.org/x/sync/errgroup"
1010

1111
"cdr.dev/slog"
12+
1213
"github.com/coder/coder/v2/coderd/database"
1314
"github.com/coder/coder/v2/coderd/database/dbauthz"
1415
)
1516

1617
const (
17-
delay = 24 * time.Hour
18+
delay = 10 * time.Minute
1819
)
1920

2021
// New creates a new periodically purging database instance.
@@ -23,37 +24,47 @@ const (
2324
// This is for cleaning up old, unused resources from the database that take up space.
2425
func New(ctx context.Context, logger slog.Logger, db database.Store) io.Closer {
2526
closed := make(chan struct{})
27+
2628
ctx, cancelFunc := context.WithCancel(ctx)
2729
//nolint:gocritic // The system purges old db records without user input.
2830
ctx = dbauthz.AsSystemRestricted(ctx)
31+
32+
// Use time.Nanosecond to force an initial tick. It will be reset to the
33+
// correct duration after executing once.
34+
ticker := time.NewTicker(time.Nanosecond)
35+
doTick := func() {
36+
defer ticker.Reset(delay)
37+
38+
var eg errgroup.Group
39+
eg.Go(func() error {
40+
return db.DeleteOldWorkspaceAgentLogs(ctx)
41+
})
42+
eg.Go(func() error {
43+
return db.DeleteOldWorkspaceAgentStats(ctx)
44+
})
45+
eg.Go(func() error {
46+
return db.DeleteOldProvisionerDaemons(ctx)
47+
})
48+
err := eg.Wait()
49+
if err != nil {
50+
if errors.Is(err, context.Canceled) {
51+
return
52+
}
53+
logger.Error(ctx, "failed to purge old database entries", slog.Error(err))
54+
}
55+
}
56+
2957
go func() {
3058
defer close(closed)
31-
32-
ticker := time.NewTicker(delay)
3359
defer ticker.Stop()
3460
for {
3561
select {
3662
case <-ctx.Done():
3763
return
3864
case <-ticker.C:
65+
ticker.Stop()
66+
doTick()
3967
}
40-
41-
var eg errgroup.Group
42-
eg.Go(func() error {
43-
return db.DeleteOldWorkspaceAgentLogs(ctx)
44-
})
45-
eg.Go(func() error {
46-
return db.DeleteOldWorkspaceAgentStats(ctx)
47-
})
48-
err := eg.Wait()
49-
if err != nil {
50-
if errors.Is(err, context.Canceled) {
51-
return
52-
}
53-
logger.Error(ctx, "failed to purge old database entries", slog.Error(err))
54-
}
55-
56-
ticker.Reset(delay)
5768
}
5869
}()
5970
return &instance{

‎coderd/database/dbpurge/dbpurge_test.go

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,23 @@ package dbpurge_test
22

33
import (
44
"context"
5+
"database/sql"
56
"testing"
7+
"time"
68

7-
"go.uber.org/goleak"
8-
9+
"github.com/google/uuid"
910
"github.com/stretchr/testify/require"
11+
"go.uber.org/goleak"
12+
"golang.org/x/exp/slices"
1013

1114
"cdr.dev/slog/sloggers/slogtest"
15+
16+
"github.com/coder/coder/v2/coderd/database"
1217
"github.com/coder/coder/v2/coderd/database/dbmem"
1318
"github.com/coder/coder/v2/coderd/database/dbpurge"
19+
"github.com/coder/coder/v2/coderd/database/dbtestutil"
20+
"github.com/coder/coder/v2/coderd/database/dbtime"
21+
"github.com/coder/coder/v2/testutil"
1422
)
1523

1624
func TestMain(m *testing.M) {
@@ -24,3 +32,72 @@ func TestPurge(t *testing.T) {
2432
err := purger.Close()
2533
require.NoError(t, err)
2634
}
35+
36+
func TestDeleteOldProvisionerDaemons(t *testing.T) {
37+
t.Parallel()
38+
39+
db, _ := dbtestutil.NewDB(t)
40+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
41+
42+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
43+
defer cancel()
44+
45+
now := dbtime.Now()
46+
47+
// given
48+
_, err := db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
49+
// Provisioner daemon created 14 days ago, and checked in just before 7 days deadline.
50+
ID: uuid.New(),
51+
Name: "external-0",
52+
Provisioners: []database.ProvisionerType{"echo"},
53+
CreatedAt: now.Add(-14 * 24 * time.Hour),
54+
UpdatedAt: sql.NullTime{Valid: true, Time: now.Add(-7 * 24 * time.Hour).Add(time.Minute)},
55+
})
56+
require.NoError(t, err)
57+
_, err = db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
58+
// Provisioner daemon created 8 days ago, and checked in last time an hour after creation.
59+
ID: uuid.New(),
60+
Name: "external-1",
61+
Provisioners: []database.ProvisionerType{"echo"},
62+
CreatedAt: now.Add(-8 * 24 * time.Hour),
63+
UpdatedAt: sql.NullTime{Valid: true, Time: now.Add(-8 * 24 * time.Hour).Add(time.Hour)},
64+
})
65+
require.NoError(t, err)
66+
_, err = db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
67+
// Provisioner daemon created 9 days ago, and never checked in.
68+
ID: uuid.New(),
69+
Name: "external-2",
70+
Provisioners: []database.ProvisionerType{"echo"},
71+
CreatedAt: now.Add(-9 * 24 * time.Hour),
72+
})
73+
require.NoError(t, err)
74+
_, err = db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
75+
// Provisioner daemon created 6 days ago, and never checked in.
76+
ID: uuid.New(),
77+
Name: "external-3",
78+
Provisioners: []database.ProvisionerType{"echo"},
79+
CreatedAt: now.Add(-6 * 24 * time.Hour),
80+
UpdatedAt: sql.NullTime{Valid: true, Time: now.Add(-6 * 24 * time.Hour)},
81+
})
82+
require.NoError(t, err)
83+
84+
// when
85+
closer := dbpurge.New(ctx, logger, db)
86+
defer closer.Close()
87+
88+
// then
89+
require.Eventually(t, func() bool {
90+
daemons, err := db.GetProvisionerDaemons(ctx)
91+
if err != nil {
92+
return false
93+
}
94+
return contains(daemons, "external-0") &&
95+
contains(daemons, "external-3")
96+
}, testutil.WaitShort, testutil.IntervalFast)
97+
}
98+
99+
func contains(daemons []database.ProvisionerDaemon, name string) bool {
100+
return slices.ContainsFunc(daemons, func(d database.ProvisionerDaemon) bool {
101+
return d.Name == name
102+
})
103+
}

‎coderd/database/querier.go

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/queries.sql.go

Lines changed: 21 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/queries/provisionerdaemons.sql

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,18 @@ INSERT INTO
1111
created_at,
1212
"name",
1313
provisioners,
14-
tags
14+
tags,
15+
updated_at
1516
)
1617
VALUES
17-
($1, $2, $3, $4, $5) RETURNING *;
18+
($1, $2, $3, $4, $5, $6) RETURNING *;
19+
20+
-- name: DeleteOldProvisionerDaemons :exec
21+
-- Delete provisioner daemons that have been created at least a week ago
22+
-- and have not connected to coderd since a week.
23+
-- A provisioner daemon with "zeroed" updated_at column indicates possible
24+
-- connectivity issues (no provisioner daemon activity since registration).
25+
DELETE FROM provisioner_daemons WHERE (
26+
(created_at < (NOW() - INTERVAL '7 days') AND updated_at IS NULL) OR
27+
(updated_at IS NOT NULL AND updated_at < (NOW() - INTERVAL '7 days'))
28+
);

0 commit comments

Comments
 (0)