Skip to content

Commit ba051e2

Browse files
matifalimatifali
authored
Merge branch 'main' into fly-wsproxy
2 parents 0057b72 + 28eca2e commit ba051e2

File tree

146 files changed

+1188
-819
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

146 files changed

+1188
-819
lines changed

‎.github/workflows/ci.yaml

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -618,6 +618,8 @@ jobs:
618618
echo "- test-go-pg: ${{ needs.test-go-pg.result }}"
619619
echo "- test-go-race: ${{ needs.test-go-race.result }}"
620620
echo "- test-js: ${{ needs.test-js.result }}"
621+
echo "- test-e2e: ${{ needs.test-e2e.result }}"
622+
echo "- offlinedocs: ${{ needs.offlinedocs.result }}"
621623
echo
622624
623625
# We allow skipped jobs to pass, but not failed or cancelled jobs.
@@ -643,21 +645,24 @@ jobs:
643645
with:
644646
fetch-depth: 0
645647

648+
- name: GHCR Login
649+
uses: docker/login-action@v3
650+
with:
651+
registry: ghcr.io
652+
username: ${{ github.actor }}
653+
password: ${{ secrets.GITHUB_TOKEN }}
654+
646655
- name: Setup Node
647656
uses: ./.github/actions/setup-node
648657

649658
- name: Setup Go
650659
uses: ./.github/actions/setup-go
651660

652-
- name: Setup sqlc
653-
uses: ./.github/actions/setup-sqlc
661+
- name: Install nfpm
662+
run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.16.0
654663

655-
- name: GHCR Login
656-
uses: docker/login-action@v3
657-
with:
658-
registry: ghcr.io
659-
username: ${{ github.actor }}
660-
password: ${{ secrets.GITHUB_TOKEN }}
664+
- name: Install zstd
665+
run: sudo apt-get install -y zstd
661666

662667
- name: Build
663668
run: |
@@ -674,6 +679,7 @@ jobs:
674679
- name: Build and Push Linux amd64 Docker Image
675680
run: |
676681
set -euxo pipefail
682+
version="$(./scripts/version.sh)"
677683
tag="main-$(echo "$version" | sed 's/+/-/g')"
678684
679685
export CODER_IMAGE_BUILD_BASE_TAG="$(CODER_IMAGE_BASE=coder-base ./scripts/image_tag.sh --version "$version")"
@@ -714,7 +720,7 @@ jobs:
714720

715721
deploy:
716722
name: "deploy"
717-
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-16vcpu-ubuntu-2204' || 'ubuntu-latest' }}
723+
runs-on: ubuntu-latest
718724
timeout-minutes: 30
719725
needs:
720726
- changes
@@ -796,12 +802,8 @@ jobs:
796802
TOKEN_SYDNEY: ${{ secrets.FLY_SYDNEY_CODER_PROXY_SESSION_TOKEN }}
797803
TOKEN_SAO_PAULO: ${{ secrets.FLY_SAO_PAULO_CODER_PROXY_SESSION_TOKEN }}
798804

799-
800-
801-
# TODO: when we remove this, instead of removing it we need to change it so it
802-
# still upgrades workspace proxies which are not deployed on K8s
803-
deploy-legacy:
804-
name: "deploy-legacy"
805+
deploy-legacy-proxies:
806+
name: "deploy-legacy-proxies"
805807
runs-on: ${{ github.repository_owner == 'coder' && 'buildjet-16vcpu-ubuntu-2204' || 'ubuntu-latest' }}
806808
timeout-minutes: 30
807809
needs: changes
@@ -857,7 +859,6 @@ jobs:
857859
858860
regions=(
859861
# gcp-region-id instance-name systemd-service-name
860-
"us-central1-a coder coder"
861862
"australia-southeast1-b coder-sydney coder-workspace-proxy"
862863
"europe-west3-c coder-europe coder-workspace-proxy"
863864
"southamerica-east1-b coder-brazil coder-workspace-proxy"

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ endif
5050
# Note, all find statements should be written with `.` or `./path` as
5151
# the search path so that these exclusions match.
5252
FIND_EXCLUSIONS= \
53-
-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' \) -prune \)
53+
-not \( \( -path '*/.git/*' -o -path './build/*' -o -path './vendor/*' -o -path './.coderv2/*' -o -path '*/node_modules/*' -o -path '*/out/*' -o -path './coderd/apidoc/*' -o -path '*/.next/*' -o -path '*/.terraform/*' \) -prune \)
5454
# Source files used for make targets, evaluated on use.
5555
GO_SRC_FILES := $(shell find . $(FIND_EXCLUSIONS) -type f -name '*.go' -not -name '*_test.go')
5656
# All the shell files in the repo, excluding ignored files.

‎coderd/database/dbauthz/dbauthz.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -791,6 +791,13 @@ func (q *querier) DeleteLicense(ctx context.Context, id int32) (int32, error) {
791791
return id, nil
792792
}
793793

794+
func (q *querier) DeleteOldProvisionerDaemons(ctx context.Context) error {
795+
if err := q.authorizeContext(ctx, rbac.ActionDelete, rbac.ResourceSystem); err != nil {
796+
return err
797+
}
798+
return q.db.DeleteOldProvisionerDaemons(ctx)
799+
}
800+
794801
func (q *querier) DeleteOldWorkspaceAgentLogs(ctx context.Context) error {
795802
if err := q.authorizeContext(ctx, rbac.ActionDelete, rbac.ResourceSystem); err != nil {
796803
return err

‎coderd/database/dbmem/dbmem.go

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1112,8 +1112,27 @@ func (q *FakeQuerier) DeleteLicense(_ context.Context, id int32) (int32, error)
11121112
return 0, sql.ErrNoRows
11131113
}
11141114

1115+
func (q *FakeQuerier) DeleteOldProvisionerDaemons(_ context.Context) error {
1116+
q.mutex.Lock()
1117+
defer q.mutex.Unlock()
1118+
1119+
now := dbtime.Now()
1120+
weekInterval := 7 * 24 * time.Hour
1121+
weekAgo := now.Add(-weekInterval)
1122+
1123+
var validDaemons []database.ProvisionerDaemon
1124+
for _, p := range q.provisionerDaemons {
1125+
if (p.CreatedAt.Before(weekAgo) && !p.UpdatedAt.Valid) || (p.UpdatedAt.Valid && p.UpdatedAt.Time.Before(weekAgo)) {
1126+
continue
1127+
}
1128+
validDaemons = append(validDaemons, p)
1129+
}
1130+
q.provisionerDaemons = validDaemons
1131+
return nil
1132+
}
1133+
11151134
func (*FakeQuerier) DeleteOldWorkspaceAgentLogs(_ context.Context) error {
1116-
// noop
1135+
// no-op
11171136
return nil
11181137
}
11191138

@@ -4845,6 +4864,7 @@ func (q *FakeQuerier) InsertProvisionerDaemon(_ context.Context, arg database.In
48454864
Name: arg.Name,
48464865
Provisioners: arg.Provisioners,
48474866
Tags: arg.Tags,
4867+
UpdatedAt: arg.UpdatedAt,
48484868
}
48494869
q.provisionerDaemons = append(q.provisionerDaemons, daemon)
48504870
return daemon, nil

‎coderd/database/dbmetrics/dbmetrics.go

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbmock/dbmock.go

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbpurge/dbpurge.go

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,13 @@ import (
99
"golang.org/x/sync/errgroup"
1010

1111
"cdr.dev/slog"
12+
1213
"github.com/coder/coder/v2/coderd/database"
1314
"github.com/coder/coder/v2/coderd/database/dbauthz"
1415
)
1516

1617
const (
17-
delay = 24 * time.Hour
18+
delay = 10 * time.Minute
1819
)
1920

2021
// New creates a new periodically purging database instance.
@@ -23,37 +24,47 @@ const (
2324
// This is for cleaning up old, unused resources from the database that take up space.
2425
func New(ctx context.Context, logger slog.Logger, db database.Store) io.Closer {
2526
closed := make(chan struct{})
27+
2628
ctx, cancelFunc := context.WithCancel(ctx)
2729
//nolint:gocritic // The system purges old db records without user input.
2830
ctx = dbauthz.AsSystemRestricted(ctx)
31+
32+
// Use time.Nanosecond to force an initial tick. It will be reset to the
33+
// correct duration after executing once.
34+
ticker := time.NewTicker(time.Nanosecond)
35+
doTick := func() {
36+
defer ticker.Reset(delay)
37+
38+
var eg errgroup.Group
39+
eg.Go(func() error {
40+
return db.DeleteOldWorkspaceAgentLogs(ctx)
41+
})
42+
eg.Go(func() error {
43+
return db.DeleteOldWorkspaceAgentStats(ctx)
44+
})
45+
eg.Go(func() error {
46+
return db.DeleteOldProvisionerDaemons(ctx)
47+
})
48+
err := eg.Wait()
49+
if err != nil {
50+
if errors.Is(err, context.Canceled) {
51+
return
52+
}
53+
logger.Error(ctx, "failed to purge old database entries", slog.Error(err))
54+
}
55+
}
56+
2957
go func() {
3058
defer close(closed)
31-
32-
ticker := time.NewTicker(delay)
3359
defer ticker.Stop()
3460
for {
3561
select {
3662
case <-ctx.Done():
3763
return
3864
case <-ticker.C:
65+
ticker.Stop()
66+
doTick()
3967
}
40-
41-
var eg errgroup.Group
42-
eg.Go(func() error {
43-
return db.DeleteOldWorkspaceAgentLogs(ctx)
44-
})
45-
eg.Go(func() error {
46-
return db.DeleteOldWorkspaceAgentStats(ctx)
47-
})
48-
err := eg.Wait()
49-
if err != nil {
50-
if errors.Is(err, context.Canceled) {
51-
return
52-
}
53-
logger.Error(ctx, "failed to purge old database entries", slog.Error(err))
54-
}
55-
56-
ticker.Reset(delay)
5768
}
5869
}()
5970
return &instance{

‎coderd/database/dbpurge/dbpurge_test.go

Lines changed: 79 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,23 @@ package dbpurge_test
22

33
import (
44
"context"
5+
"database/sql"
56
"testing"
7+
"time"
68

7-
"go.uber.org/goleak"
8-
9+
"github.com/google/uuid"
910
"github.com/stretchr/testify/require"
11+
"go.uber.org/goleak"
12+
"golang.org/x/exp/slices"
1013

1114
"cdr.dev/slog/sloggers/slogtest"
15+
16+
"github.com/coder/coder/v2/coderd/database"
1217
"github.com/coder/coder/v2/coderd/database/dbmem"
1318
"github.com/coder/coder/v2/coderd/database/dbpurge"
19+
"github.com/coder/coder/v2/coderd/database/dbtestutil"
20+
"github.com/coder/coder/v2/coderd/database/dbtime"
21+
"github.com/coder/coder/v2/testutil"
1422
)
1523

1624
func TestMain(m *testing.M) {
@@ -24,3 +32,72 @@ func TestPurge(t *testing.T) {
2432
err := purger.Close()
2533
require.NoError(t, err)
2634
}
35+
36+
func TestDeleteOldProvisionerDaemons(t *testing.T) {
37+
t.Parallel()
38+
39+
db, _ := dbtestutil.NewDB(t)
40+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
41+
42+
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
43+
defer cancel()
44+
45+
now := dbtime.Now()
46+
47+
// given
48+
_, err := db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
49+
// Provisioner daemon created 14 days ago, and checked in just before 7 days deadline.
50+
ID: uuid.New(),
51+
Name: "external-0",
52+
Provisioners: []database.ProvisionerType{"echo"},
53+
CreatedAt: now.Add(-14 * 24 * time.Hour),
54+
UpdatedAt: sql.NullTime{Valid: true, Time: now.Add(-7 * 24 * time.Hour).Add(time.Minute)},
55+
})
56+
require.NoError(t, err)
57+
_, err = db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
58+
// Provisioner daemon created 8 days ago, and checked in last time an hour after creation.
59+
ID: uuid.New(),
60+
Name: "external-1",
61+
Provisioners: []database.ProvisionerType{"echo"},
62+
CreatedAt: now.Add(-8 * 24 * time.Hour),
63+
UpdatedAt: sql.NullTime{Valid: true, Time: now.Add(-8 * 24 * time.Hour).Add(time.Hour)},
64+
})
65+
require.NoError(t, err)
66+
_, err = db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
67+
// Provisioner daemon created 9 days ago, and never checked in.
68+
ID: uuid.New(),
69+
Name: "external-2",
70+
Provisioners: []database.ProvisionerType{"echo"},
71+
CreatedAt: now.Add(-9 * 24 * time.Hour),
72+
})
73+
require.NoError(t, err)
74+
_, err = db.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{
75+
// Provisioner daemon created 6 days ago, and never checked in.
76+
ID: uuid.New(),
77+
Name: "external-3",
78+
Provisioners: []database.ProvisionerType{"echo"},
79+
CreatedAt: now.Add(-6 * 24 * time.Hour),
80+
UpdatedAt: sql.NullTime{Valid: true, Time: now.Add(-6 * 24 * time.Hour)},
81+
})
82+
require.NoError(t, err)
83+
84+
// when
85+
closer := dbpurge.New(ctx, logger, db)
86+
defer closer.Close()
87+
88+
// then
89+
require.Eventually(t, func() bool {
90+
daemons, err := db.GetProvisionerDaemons(ctx)
91+
if err != nil {
92+
return false
93+
}
94+
return contains(daemons, "external-0") &&
95+
contains(daemons, "external-3")
96+
}, testutil.WaitShort, testutil.IntervalFast)
97+
}
98+
99+
func contains(daemons []database.ProvisionerDaemon, name string) bool {
100+
return slices.ContainsFunc(daemons, func(d database.ProvisionerDaemon) bool {
101+
return d.Name == name
102+
})
103+
}

‎coderd/database/dump.sql

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/migrations/000018_provisioner_job_type_dry_run.up.sql

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,11 @@
1-
ALTER TYPE provisioner_job_type
2-
ADD VALUE IF NOT EXISTS 'template_version_dry_run';
1+
CREATE TYPE new_provisioner_job_type AS ENUM (
2+
'template_version_import',
3+
'workspace_build',
4+
'template_version_dry_run'
5+
);
6+
7+
ALTER TABLE provisioner_jobs
8+
ALTER COLUMN "type" TYPE new_provisioner_job_type USING ("type"::text::new_provisioner_job_type);
9+
10+
DROP TYPE provisioner_job_type;
11+
ALTER TYPE new_provisioner_job_type RENAME TO provisioner_job_type;

‎coderd/database/migrations/000030_template_version_created_by.up.sql

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
BEGIN;
21

32
ALTER TABLE ONLY template_versions ADD COLUMN IF NOT EXISTS created_by uuid REFERENCES users (id) ON DELETE RESTRICT;
43

@@ -12,5 +11,3 @@ SET
1211
)
1312
WHERE
1413
created_by IS NULL;
15-
16-
COMMIT;

0 commit comments

Comments
 (0)