Skip to content

Commit cf8be4e

Browse files
deansheatherdeansheather
authored
feat: add resume support to coordinator connections (coder#14234)
1 parent 0b2ba96 commit cf8be4e

32 files changed

+1706
-465
lines changed

‎cli/server.go

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ import (
5656
"cdr.dev/slog"
5757
"cdr.dev/slog/sloggers/sloghuman"
5858
"github.com/coder/pretty"
59+
"github.com/coder/quartz"
5960
"github.com/coder/retry"
6061
"github.com/coder/serpent"
6162
"github.com/coder/wgtunnel/tunnelsdk"
@@ -791,18 +792,26 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
791792
}
792793
}
793794

794-
keyBytes, err := hex.DecodeString(oauthSigningKeyStr)
795+
oauthKeyBytes, err := hex.DecodeString(oauthSigningKeyStr)
795796
if err != nil {
796797
return xerrors.Errorf("decode oauth signing key from database: %w", err)
797798
}
798-
if len(keyBytes) != len(options.OAuthSigningKey) {
799-
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(keyBytes))
799+
if len(oauthKeyBytes) != len(options.OAuthSigningKey) {
800+
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(oauthKeyBytes))
800801
}
801-
copy(options.OAuthSigningKey[:], keyBytes)
802+
copy(options.OAuthSigningKey[:], oauthKeyBytes)
802803
if options.OAuthSigningKey == [32]byte{} {
803804
return xerrors.Errorf("oauth signing key in database is empty")
804805
}
805806

807+
// Read the coordinator resume token signing key from the
808+
// database.
809+
resumeTokenKey, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, tx)
810+
if err != nil {
811+
return xerrors.Errorf("get coordinator resume token key from database: %w", err)
812+
}
813+
options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeTokenKey, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry)
814+
806815
return nil
807816
}, nil)
808817
if err != nil {

‎coderd/coderd.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,9 @@ type Options struct {
182182
// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
183183
// workspace applications. It consists of both a signing and encryption key.
184184
AppSecurityKey workspaceapps.SecurityKey
185+
// CoordinatorResumeTokenProvider is used to provide and validate resume
186+
// tokens issued by and passed to the coordinator DRPC API.
187+
CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider
185188

186189
HealthcheckFunc func(ctx context.Context, apiKey string) *healthsdk.HealthcheckReport
187190
HealthcheckTimeout time.Duration
@@ -584,12 +587,16 @@ func New(options *Options) *API {
584587
api.Options.NetworkTelemetryBatchMaxSize,
585588
api.handleNetworkTelemetry,
586589
)
590+
if options.CoordinatorResumeTokenProvider == nil {
591+
panic("CoordinatorResumeTokenProvider is nil")
592+
}
587593
api.TailnetClientService, err = tailnet.NewClientService(tailnet.ClientServiceOptions{
588594
Logger: api.Logger.Named("tailnetclient"),
589595
CoordPtr: &api.TailnetCoordinator,
590596
DERPMapUpdateFrequency: api.Options.DERPMapUpdateFrequency,
591597
DERPMapFn: api.DERPMap,
592598
NetworkTelemetryHandler: api.NetworkTelemetryBatcher.Handler,
599+
ResumeTokenProvider: api.Options.CoordinatorResumeTokenProvider,
593600
})
594601
if err != nil {
595602
api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err))
@@ -614,6 +621,9 @@ func New(options *Options) *API {
614621
options.WorkspaceAppsStatsCollectorOptions.Reporter = api.statsReporter
615622
}
616623

624+
if options.AppSecurityKey.IsZero() {
625+
api.Logger.Fatal(api.ctx, "app security key cannot be zero")
626+
}
617627
api.workspaceAppServer = &workspaceapps.Server{
618628
Logger: workspaceAppsLogger,
619629

‎coderd/coderdtest/coderdtest.go

Lines changed: 24 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -96,25 +96,26 @@ type Options struct {
9696
// AccessURL denotes a custom access URL. By default we use the httptest
9797
// server's URL. Setting this may result in unexpected behavior (especially
9898
// with running agents).
99-
AccessURL *url.URL
100-
AppHostname string
101-
AWSCertificates awsidentity.Certificates
102-
Authorizer rbac.Authorizer
103-
AzureCertificates x509.VerifyOptions
104-
GithubOAuth2Config *coderd.GithubOAuth2Config
105-
RealIPConfig *httpmw.RealIPConfig
106-
OIDCConfig *coderd.OIDCConfig
107-
GoogleTokenValidator *idtoken.Validator
108-
SSHKeygenAlgorithm gitsshkey.Algorithm
109-
AutobuildTicker <-chan time.Time
110-
AutobuildStats chan<- autobuild.Stats
111-
Auditor audit.Auditor
112-
TLSCertificates []tls.Certificate
113-
ExternalAuthConfigs []*externalauth.Config
114-
TrialGenerator func(ctx context.Context, body codersdk.LicensorTrialRequest) error
115-
RefreshEntitlements func(ctx context.Context) error
116-
TemplateScheduleStore schedule.TemplateScheduleStore
117-
Coordinator tailnet.Coordinator
99+
AccessURL *url.URL
100+
AppHostname string
101+
AWSCertificates awsidentity.Certificates
102+
Authorizer rbac.Authorizer
103+
AzureCertificates x509.VerifyOptions
104+
GithubOAuth2Config *coderd.GithubOAuth2Config
105+
RealIPConfig *httpmw.RealIPConfig
106+
OIDCConfig *coderd.OIDCConfig
107+
GoogleTokenValidator *idtoken.Validator
108+
SSHKeygenAlgorithm gitsshkey.Algorithm
109+
AutobuildTicker <-chan time.Time
110+
AutobuildStats chan<- autobuild.Stats
111+
Auditor audit.Auditor
112+
TLSCertificates []tls.Certificate
113+
ExternalAuthConfigs []*externalauth.Config
114+
TrialGenerator func(ctx context.Context, body codersdk.LicensorTrialRequest) error
115+
RefreshEntitlements func(ctx context.Context) error
116+
TemplateScheduleStore schedule.TemplateScheduleStore
117+
Coordinator tailnet.Coordinator
118+
CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider
118119

119120
HealthcheckFunc func(ctx context.Context, apiKey string) *healthsdk.HealthcheckReport
120121
HealthcheckTimeout time.Duration
@@ -240,6 +241,9 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
240241
if options.Database == nil {
241242
options.Database, options.Pubsub = dbtestutil.NewDB(t)
242243
}
244+
if options.CoordinatorResumeTokenProvider == nil {
245+
options.CoordinatorResumeTokenProvider = tailnet.NewInsecureTestResumeTokenProvider()
246+
}
243247

244248
if options.NotificationsEnqueuer == nil {
245249
options.NotificationsEnqueuer = new(testutil.FakeNotificationsEnqueuer)
@@ -492,6 +496,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
492496
TailnetCoordinator: options.Coordinator,
493497
BaseDERPMap: derpMap,
494498
DERPMapUpdateFrequency: 150 * time.Millisecond,
499+
CoordinatorResumeTokenProvider: options.CoordinatorResumeTokenProvider,
495500
MetricsCacheRefreshInterval: options.MetricsCacheRefreshInterval,
496501
AgentStatsRefreshInterval: options.AgentStatsRefreshInterval,
497502
DeploymentValues: options.DeploymentValues,

‎coderd/database/dbauthz/dbauthz.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1332,7 +1332,9 @@ func (q *querier) GetAnnouncementBanners(ctx context.Context) (string, error) {
13321332
}
13331333

13341334
func (q *querier) GetAppSecurityKey(ctx context.Context) (string, error) {
1335-
// No authz checks
1335+
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
1336+
return "", err
1337+
}
13361338
return q.db.GetAppSecurityKey(ctx)
13371339
}
13381340

@@ -1364,6 +1366,13 @@ func (q *querier) GetAuthorizationUserRoles(ctx context.Context, userID uuid.UUI
13641366
return q.db.GetAuthorizationUserRoles(ctx, userID)
13651367
}
13661368

1369+
func (q *querier) GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) {
1370+
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
1371+
return "", err
1372+
}
1373+
return q.db.GetCoordinatorResumeTokenSigningKey(ctx)
1374+
}
1375+
13671376
func (q *querier) GetDBCryptKeys(ctx context.Context) ([]database.DBCryptKey, error) {
13681377
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
13691378
return nil, err
@@ -3792,7 +3801,9 @@ func (q *querier) UpsertAnnouncementBanners(ctx context.Context, value string) e
37923801
}
37933802

37943803
func (q *querier) UpsertAppSecurityKey(ctx context.Context, data string) error {
3795-
// No authz checks as this is done during startup
3804+
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
3805+
return err
3806+
}
37963807
return q.db.UpsertAppSecurityKey(ctx, data)
37973808
}
37983809

@@ -3803,6 +3814,13 @@ func (q *querier) UpsertApplicationName(ctx context.Context, value string) error
38033814
return q.db.UpsertApplicationName(ctx, value)
38043815
}
38053816

3817+
func (q *querier) UpsertCoordinatorResumeTokenSigningKey(ctx context.Context, value string) error {
3818+
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
3819+
return err
3820+
}
3821+
return q.db.UpsertCoordinatorResumeTokenSigningKey(ctx, value)
3822+
}
3823+
38063824
func (q *querier) UpsertDefaultProxy(ctx context.Context, arg database.UpsertDefaultProxyParams) error {
38073825
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
38083826
return err

‎coderd/database/dbauthz/dbauthz_test.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2531,10 +2531,10 @@ func (s *MethodTestSuite) TestSystemFunctions() {
25312531
check.Args(int32(0)).Asserts(rbac.ResourceSystem, policy.ActionRead)
25322532
}))
25332533
s.Run("GetAppSecurityKey", s.Subtest(func(db database.Store, check *expects) {
2534-
check.Args().Asserts()
2534+
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
25352535
}))
25362536
s.Run("UpsertAppSecurityKey", s.Subtest(func(db database.Store, check *expects) {
2537-
check.Args("").Asserts()
2537+
check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
25382538
}))
25392539
s.Run("GetApplicationName", s.Subtest(func(db database.Store, check *expects) {
25402540
db.UpsertApplicationName(context.Background(), "foo")
@@ -2574,6 +2574,13 @@ func (s *MethodTestSuite) TestSystemFunctions() {
25742574
db.UpsertOAuthSigningKey(context.Background(), "foo")
25752575
check.Args().Asserts(rbac.ResourceSystem, policy.ActionUpdate)
25762576
}))
2577+
s.Run("UpsertCoordinatorResumeTokenSigningKey", s.Subtest(func(db database.Store, check *expects) {
2578+
check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
2579+
}))
2580+
s.Run("GetCoordinatorResumeTokenSigningKey", s.Subtest(func(db database.Store, check *expects) {
2581+
db.UpsertCoordinatorResumeTokenSigningKey(context.Background(), "foo")
2582+
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
2583+
}))
25772584
s.Run("InsertMissingGroups", s.Subtest(func(db database.Store, check *expects) {
25782585
check.Args(database.InsertMissingGroupsParams{}).Asserts(rbac.ResourceSystem, policy.ActionCreate).Errors(errMatchAny)
25792586
}))

‎coderd/database/dbmem/dbmem.go

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -196,20 +196,21 @@ type data struct {
196196
customRoles []database.CustomRole
197197
// Locks is a map of lock names. Any keys within the map are currently
198198
// locked.
199-
locks map[int64]struct{}
200-
deploymentID string
201-
derpMeshKey string
202-
lastUpdateCheck []byte
203-
announcementBanners []byte
204-
healthSettings []byte
205-
notificationsSettings []byte
206-
applicationName string
207-
logoURL string
208-
appSecurityKey string
209-
oauthSigningKey string
210-
lastLicenseID int32
211-
defaultProxyDisplayName string
212-
defaultProxyIconURL string
199+
locks map[int64]struct{}
200+
deploymentID string
201+
derpMeshKey string
202+
lastUpdateCheck []byte
203+
announcementBanners []byte
204+
healthSettings []byte
205+
notificationsSettings []byte
206+
applicationName string
207+
logoURL string
208+
appSecurityKey string
209+
oauthSigningKey string
210+
coordinatorResumeTokenSigningKey string
211+
lastLicenseID int32
212+
defaultProxyDisplayName string
213+
defaultProxyIconURL string
213214
}
214215

215216
func validateDatabaseTypeWithValid(v reflect.Value) (handled bool, err error) {
@@ -2222,6 +2223,15 @@ func (q *FakeQuerier) GetAuthorizationUserRoles(_ context.Context, userID uuid.U
22222223
}, nil
22232224
}
22242225

2226+
func (q *FakeQuerier) GetCoordinatorResumeTokenSigningKey(_ context.Context) (string, error) {
2227+
q.mutex.RLock()
2228+
defer q.mutex.RUnlock()
2229+
if q.coordinatorResumeTokenSigningKey == "" {
2230+
return "", sql.ErrNoRows
2231+
}
2232+
return q.coordinatorResumeTokenSigningKey, nil
2233+
}
2234+
22252235
func (q *FakeQuerier) GetDBCryptKeys(_ context.Context) ([]database.DBCryptKey, error) {
22262236
q.mutex.RLock()
22272237
defer q.mutex.RUnlock()
@@ -8942,6 +8952,14 @@ func (q *FakeQuerier) UpsertApplicationName(_ context.Context, data string) erro
89428952
return nil
89438953
}
89448954

8955+
func (q *FakeQuerier) UpsertCoordinatorResumeTokenSigningKey(_ context.Context, value string) error {
8956+
q.mutex.Lock()
8957+
defer q.mutex.Unlock()
8958+
8959+
q.coordinatorResumeTokenSigningKey = value
8960+
return nil
8961+
}
8962+
89458963
func (q *FakeQuerier) UpsertDefaultProxy(_ context.Context, arg database.UpsertDefaultProxyParams) error {
89468964
q.defaultProxyDisplayName = arg.DisplayName
89478965
q.defaultProxyIconURL = arg.IconUrl

‎coderd/database/dbmetrics/dbmetrics.go

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbmock/dbmock.go

Lines changed: 29 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/querier.go

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)