From f349d3ad2ff0bd786abbccced627ee8ffdd253a2 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 7 Dec 2023 11:19:17 +0100 Subject: [PATCH 1/4] dbmem impl --- coderd/database/dbmem/dbmem.go | 17 +++++++++++++++-- coderd/database/dbpurge/dbpurge_test.go | 6 ++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index de5700bd2ac9d..69da49c464548 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1180,8 +1180,21 @@ func (q *FakeQuerier) DeleteOldWorkspaceAgentLogs(_ context.Context) error { return nil } -func (*FakeQuerier) DeleteOldWorkspaceAgentStats(_ context.Context) error { - // no-op +func (q *FakeQuerier) DeleteOldWorkspaceAgentStats(_ context.Context) error { + q.mutex.Lock() + defer q.mutex.Unlock() + + now := dbtime.Now() + sixMonthInterval := 6 * 30 * 24 * time.Hour + sixMonthsAgo := now.Add(-sixMonthInterval) + + var validStats []database.WorkspaceAgentStat + for _, stat := range q.workspaceAgentStats { + if !stat.CreatedAt.Before(sixMonthsAgo) { + validStats = append(validStats, stat) + } + } + q.workspaceAgentStats = validStats return nil } diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 8f5eb0f68ebcc..26c35f8783904 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -34,6 +34,12 @@ func TestPurge(t *testing.T) { require.NoError(t, err) } +func TestDeleteOldWorkspaceAgentStats(t *testing.T) { + t.Parallel() + + // TODO +} + func TestDeleteOldWorkspaceAgentLogs(t *testing.T) { t.Parallel() From e813d9f5faea4b4d9208f920fd6ebf2546fba2b3 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 7 Dec 2023 12:51:50 +0100 Subject: [PATCH 2/4] go impl --- coderd/database/dbmem/dbmem.go | 5 +-- coderd/database/dbpurge/dbpurge_test.go | 46 ++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/coderd/database/dbmem/dbmem.go b/coderd/database/dbmem/dbmem.go index 69da49c464548..fe45487e4402e 100644 --- a/coderd/database/dbmem/dbmem.go +++ b/coderd/database/dbmem/dbmem.go @@ -1190,9 +1190,10 @@ func (q *FakeQuerier) DeleteOldWorkspaceAgentStats(_ context.Context) error { var validStats []database.WorkspaceAgentStat for _, stat := range q.workspaceAgentStats { - if !stat.CreatedAt.Before(sixMonthsAgo) { - validStats = append(validStats, stat) + if stat.CreatedAt.Before(sixMonthsAgo) { + continue } + validStats = append(validStats, stat) } q.workspaceAgentStats = validStats return nil diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 26c35f8783904..52d6617e54b8a 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -37,7 +37,51 @@ func TestPurge(t *testing.T) { func TestDeleteOldWorkspaceAgentStats(t *testing.T) { t.Parallel() - // TODO + db, _ := dbtestutil.NewDB(t) + logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}) + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort) + defer cancel() + + now := dbtime.Now() + + // given + // Let's use RxBytes to identify stat entries. + // Stat inserted 6 months + 1 hour ago, should be deleted. + first := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ + CreatedAt: now.Add(-6*30*24*time.Hour - time.Hour), + ConnectionMedianLatencyMS: 1, + RxBytes: 1, + }) + + // Stat inserted 6 months - 1 hour ago, should not be deleted. + second := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ + CreatedAt: now.Add(-5*30*24*time.Hour + time.Hour), + ConnectionMedianLatencyMS: 1, + RxBytes: 3, + }) + + // when + closer := dbpurge.New(ctx, logger, db) + defer closer.Close() + + // then + require.Eventually(t, func() bool { + // Query all stats created not earlier than 7 months ago + stats, err := db.GetWorkspaceAgentStats(ctx, now.Add(-7*30*24*time.Hour)) + if err != nil { + return false + } + return !containsWorkspaceAgentStat(stats, first) && + containsWorkspaceAgentStat(stats, second) + }, testutil.WaitShort, testutil.IntervalFast) + +} + +func containsWorkspaceAgentStat(stats []database.GetWorkspaceAgentStatsRow, needle database.WorkspaceAgentStat) bool { + return slices.ContainsFunc(stats, func(s database.GetWorkspaceAgentStatsRow) bool { + return s.WorkspaceRxBytes == needle.RxBytes + }) } func TestDeleteOldWorkspaceAgentLogs(t *testing.T) { From a20c9eb4fd671f730e4275c3bd3732cf9e52e5e4 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 7 Dec 2023 13:16:25 +0100 Subject: [PATCH 3/4] WIP --- coderd/database/dbpurge/dbpurge_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 52d6617e54b8a..60c429f09a08f 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -66,16 +66,17 @@ func TestDeleteOldWorkspaceAgentStats(t *testing.T) { defer closer.Close() // then + var stats []database.GetWorkspaceAgentStatsRow + var err error require.Eventually(t, func() bool { // Query all stats created not earlier than 7 months ago - stats, err := db.GetWorkspaceAgentStats(ctx, now.Add(-7*30*24*time.Hour)) + stats, err = db.GetWorkspaceAgentStats(ctx, now.Add(-7*30*24*time.Hour)) if err != nil { return false } return !containsWorkspaceAgentStat(stats, first) && containsWorkspaceAgentStat(stats, second) - }, testutil.WaitShort, testutil.IntervalFast) - + }, testutil.WaitShort, testutil.IntervalFast, stats) } func containsWorkspaceAgentStat(stats []database.GetWorkspaceAgentStatsRow, needle database.WorkspaceAgentStat) bool { From c838c6658034fbac1bdb385d1c621ad63a27b43c Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 7 Dec 2023 13:37:54 +0100 Subject: [PATCH 4/4] fix --- coderd/database/dbpurge/dbpurge_test.go | 4 ++-- coderd/database/queries.sql.go | 2 +- coderd/database/queries/workspaceagentstats.sql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coderd/database/dbpurge/dbpurge_test.go b/coderd/database/dbpurge/dbpurge_test.go index 60c429f09a08f..bab7d0ffa6e96 100644 --- a/coderd/database/dbpurge/dbpurge_test.go +++ b/coderd/database/dbpurge/dbpurge_test.go @@ -51,14 +51,14 @@ func TestDeleteOldWorkspaceAgentStats(t *testing.T) { first := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ CreatedAt: now.Add(-6*30*24*time.Hour - time.Hour), ConnectionMedianLatencyMS: 1, - RxBytes: 1, + RxBytes: 1111, }) // Stat inserted 6 months - 1 hour ago, should not be deleted. second := dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{ CreatedAt: now.Add(-5*30*24*time.Hour + time.Hour), ConnectionMedianLatencyMS: 1, - RxBytes: 3, + RxBytes: 2222, }) // when diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index fb7b15cf26866..9009eea885b02 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8672,7 +8672,7 @@ func (q *sqlQuerier) UpdateWorkspaceAgentStartupByID(ctx context.Context, arg Up } const deleteOldWorkspaceAgentStats = `-- name: DeleteOldWorkspaceAgentStats :exec -DELETE FROM workspace_agent_stats WHERE created_at < NOW() - INTERVAL '6 months' +DELETE FROM workspace_agent_stats WHERE created_at < NOW() - INTERVAL '180 days' ` func (q *sqlQuerier) DeleteOldWorkspaceAgentStats(ctx context.Context) error { diff --git a/coderd/database/queries/workspaceagentstats.sql b/coderd/database/queries/workspaceagentstats.sql index d199f3617acbf..cf059121dec77 100644 --- a/coderd/database/queries/workspaceagentstats.sql +++ b/coderd/database/queries/workspaceagentstats.sql @@ -90,7 +90,7 @@ ORDER BY date ASC; -- name: DeleteOldWorkspaceAgentStats :exec -DELETE FROM workspace_agent_stats WHERE created_at < NOW() - INTERVAL '6 months'; +DELETE FROM workspace_agent_stats WHERE created_at < NOW() - INTERVAL '180 days'; -- name: GetDeploymentWorkspaceAgentStats :one WITH agent_stats AS (