Skip to content

feat(coderd/database/dbpurge): retain most recent agent build logs #14460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Aug 30, 2024
Merged

feat(coderd/database/dbpurge): retain most recent agent build logs #14460

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9451962
chore: improve coverage of dbpurge tests for DeleteOldWorkspaceAgentLogs
johnstcn Aug 29, 2024
0c14a3e
feat(dbpurge): update DeleteOldWorkspaceAgentLogs query to keep lates…
johnstcn Aug 29, 2024
0af051e
remove unnecessary columns from CTE
johnstcn Aug 29, 2024
dd97358
fix test
johnstcn Aug 29, 2024
2017434
implement dbmem
johnstcn Aug 29, 2024
725eeb0
ensure that ticks from quartz are dbtimey
johnstcn Aug 29, 2024
591dd91
extract awaitDoTick() test helper
johnstcn Aug 30, 2024
0f8ab25
address race condition in initial tick
johnstcn Aug 30, 2024
5c77b5e
fix double-tick startup race
johnstcn Aug 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
implement dbmem
  • Loading branch information
@johnstcn
johnstcn committed Aug 30, 2024
commit 2017434b6d29c74d0a6f9cf1cabee6ed979eb178
91 changes: 81 additions & 10 deletions coderd/database/dbmem/dbmem.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1710,19 +1710,90 @@ func (q *FakeQuerier) DeleteOldWorkspaceAgentLogs(_ context.Context, threshold t
q.mutex.Lock()
defer q.mutex.Unlock()

var validLogs []database.WorkspaceAgentLog
for _, log := range q.workspaceAgentLogs {
var toBeDeleted bool
for _, agent := range q.workspaceAgents {
if agent.ID == log.AgentID && agent.LastConnectedAt.Valid && agent.LastConnectedAt.Time.Before(threshold) {
toBeDeleted = true
break
}
/*
WITH
latest_builds AS (
SELECT
workspace_id, max(build_number) AS max_build_number
FROM
workspace_builds
GROUP BY
workspace_id
),
*/
latestBuilds := make(map[uuid.UUID]int32)
for _, wb := range q.workspaceBuilds {
if lastBuildNumber, found := latestBuilds[wb.WorkspaceID]; found && lastBuildNumber > wb.BuildNumber {
continue
}
// not found or newer build number
latestBuilds[wb.WorkspaceID] = wb.BuildNumber
}

if !toBeDeleted {
validLogs = append(validLogs, log)
/*
old_agents AS (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is really nice!

I'm concerned it'll go stale...
Although it could be argued all the business logic duplicated in this file is stale the second it's written.
We really need to get rid of dbmem.

SELECT
wa.id
FROM
workspace_agents AS wa
JOIN
workspace_resources AS wr
ON
wa.resource_id = wr.id
JOIN
workspace_builds AS wb
ON
wb.job_id = wr.job_id
LEFT JOIN
latest_builds
ON
latest_builds.workspace_id = wb.workspace_id
AND
latest_builds.max_build_number = wb.build_number
WHERE
-- Filter out the latest builds for each workspace.
latest_builds.workspace_id IS NULL
AND CASE
-- If the last time the agent connected was before @threshold
WHEN wa.last_connected_at IS NOT NULL THEN
wa.last_connected_at < @threshold :: timestamptz
-- The agent never connected, and was created before @threshold
ELSE wa.created_at < @threshold :: timestamptz
END
)
*/
oldAgents := make(map[uuid.UUID]struct{})
for _, wa := range q.workspaceAgents {
for _, wr := range q.workspaceResources {
if wr.ID != wa.ResourceID {
continue
}
for _, wb := range q.workspaceBuilds {
if wb.JobID != wr.JobID {
continue
}
latestBuildNumber, found := latestBuilds[wb.WorkspaceID]
if !found {
panic("workspaceBuilds got modified somehow while q was locked! This is a bug in dbmem!")
}
if latestBuildNumber == wb.BuildNumber {
continue
}
if wa.LastConnectedAt.Valid && wa.LastConnectedAt.Time.Before(threshold) || wa.CreatedAt.Before(threshold) {
oldAgents[wa.ID] = struct{}{}
}
}
}
}
/*
DELETE FROM workspace_agent_logs WHERE agent_id IN (SELECT id FROM old_agents);
*/
var validLogs []database.WorkspaceAgentLog
for _, log := range q.workspaceAgentLogs {
if _, found := oldAgents[log.AgentID]; found {
continue
}
validLogs = append(validLogs, log)
}
q.workspaceAgentLogs = validLogs
return nil
Expand Down