Skip to content

feat: add resume support to coordinator connections #14234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 20, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions cli/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ import (
"cdr.dev/slog"
"cdr.dev/slog/sloggers/sloghuman"
"github.com/coder/pretty"
"github.com/coder/quartz"
"github.com/coder/retry"
"github.com/coder/serpent"
"github.com/coder/wgtunnel/tunnelsdk"
Expand Down Expand Up @@ -791,18 +792,26 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
}
}

keyBytes, err := hex.DecodeString(oauthSigningKeyStr)
oauthKeyBytes, err := hex.DecodeString(oauthSigningKeyStr)
if err != nil {
return xerrors.Errorf("decode oauth signing key from database: %w", err)
}
if len(keyBytes) != len(options.OAuthSigningKey) {
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(keyBytes))
if len(oauthKeyBytes) != len(options.OAuthSigningKey) {
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(oauthKeyBytes))
}
copy(options.OAuthSigningKey[:], keyBytes)
copy(options.OAuthSigningKey[:], oauthKeyBytes)
if options.OAuthSigningKey == [32]byte{} {
return xerrors.Errorf("oauth signing key in database is empty")
}

// Read the coordinator resume token signing key from the
// database.
resumeTokenKey, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, tx)
if err != nil {
return xerrors.Errorf("get coordinator resume token key from database: %w", err)
}
options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeTokenKey, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry)

return nil
}, nil)
if err != nil {
Expand Down
10 changes: 10 additions & 0 deletions coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,9 @@ type Options struct {
// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
// workspace applications. It consists of both a signing and encryption key.
AppSecurityKey workspaceapps.SecurityKey
// CoordinatorResumeTokenProvider is used to provide and validate resume
// tokens issued by and passed to the coordinator DRPC API.
CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider

HealthcheckFunc func(ctx context.Context, apiKey string) *healthsdk.HealthcheckReport
HealthcheckTimeout time.Duration
Expand Down Expand Up @@ -584,12 +587,16 @@ func New(options *Options) *API {
api.Options.NetworkTelemetryBatchMaxSize,
api.handleNetworkTelemetry,
)
if options.CoordinatorResumeTokenProvider == nil {
panic("CoordinatorResumeTokenProvider is nil")
}
api.TailnetClientService, err = tailnet.NewClientService(tailnet.ClientServiceOptions{
Logger: api.Logger.Named("tailnetclient"),
CoordPtr: &api.TailnetCoordinator,
DERPMapUpdateFrequency: api.Options.DERPMapUpdateFrequency,
DERPMapFn: api.DERPMap,
NetworkTelemetryHandler: api.NetworkTelemetryBatcher.Handler,
ResumeTokenProvider: api.Options.CoordinatorResumeTokenProvider,
})
if err != nil {
api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err))
Expand All @@ -614,6 +621,9 @@ func New(options *Options) *API {
options.WorkspaceAppsStatsCollectorOptions.Reporter = api.statsReporter
}

if options.AppSecurityKey.IsZero() {
api.Logger.Fatal(api.ctx, "app security key cannot be zero")
}
api.workspaceAppServer = &workspaceapps.Server{
Logger: workspaceAppsLogger,

Expand Down
43 changes: 24 additions & 19 deletions coderd/coderdtest/coderdtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,25 +96,26 @@ type Options struct {
// AccessURL denotes a custom access URL. By default we use the httptest
// server's URL. Setting this may result in unexpected behavior (especially
// with running agents).
AccessURL *url.URL
AppHostname string
AWSCertificates awsidentity.Certificates
Authorizer rbac.Authorizer
AzureCertificates x509.VerifyOptions
GithubOAuth2Config *coderd.GithubOAuth2Config
RealIPConfig *httpmw.RealIPConfig
OIDCConfig *coderd.OIDCConfig
GoogleTokenValidator *idtoken.Validator
SSHKeygenAlgorithm gitsshkey.Algorithm
AutobuildTicker <-chan time.Time
AutobuildStats chan<- autobuild.Stats
Auditor audit.Auditor
TLSCertificates []tls.Certificate
ExternalAuthConfigs []*externalauth.Config
TrialGenerator func(ctx context.Context, body codersdk.LicensorTrialRequest) error
RefreshEntitlements func(ctx context.Context) error
TemplateScheduleStore schedule.TemplateScheduleStore
Coordinator tailnet.Coordinator
AccessURL *url.URL
AppHostname string
AWSCertificates awsidentity.Certificates
Authorizer rbac.Authorizer
AzureCertificates x509.VerifyOptions
GithubOAuth2Config *coderd.GithubOAuth2Config
RealIPConfig *httpmw.RealIPConfig
OIDCConfig *coderd.OIDCConfig
GoogleTokenValidator *idtoken.Validator
SSHKeygenAlgorithm gitsshkey.Algorithm
AutobuildTicker <-chan time.Time
AutobuildStats chan<- autobuild.Stats
Auditor audit.Auditor
TLSCertificates []tls.Certificate
ExternalAuthConfigs []*externalauth.Config
TrialGenerator func(ctx context.Context, body codersdk.LicensorTrialRequest) error
RefreshEntitlements func(ctx context.Context) error
TemplateScheduleStore schedule.TemplateScheduleStore
Coordinator tailnet.Coordinator
CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider

HealthcheckFunc func(ctx context.Context, apiKey string) *healthsdk.HealthcheckReport
HealthcheckTimeout time.Duration
Expand Down Expand Up @@ -240,6 +241,9 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
if options.Database == nil {
options.Database, options.Pubsub = dbtestutil.NewDB(t)
}
if options.CoordinatorResumeTokenProvider == nil {
options.CoordinatorResumeTokenProvider = tailnet.NewInsecureTestResumeTokenProvider()
}

if options.NotificationsEnqueuer == nil {
options.NotificationsEnqueuer = new(testutil.FakeNotificationsEnqueuer)
Expand Down Expand Up @@ -492,6 +496,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
TailnetCoordinator: options.Coordinator,
BaseDERPMap: derpMap,
DERPMapUpdateFrequency: 150 * time.Millisecond,
CoordinatorResumeTokenProvider: options.CoordinatorResumeTokenProvider,
MetricsCacheRefreshInterval: options.MetricsCacheRefreshInterval,
AgentStatsRefreshInterval: options.AgentStatsRefreshInterval,
DeploymentValues: options.DeploymentValues,
Expand Down
22 changes: 20 additions & 2 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -1332,7 +1332,9 @@ func (q *querier) GetAnnouncementBanners(ctx context.Context) (string, error) {
}

func (q *querier) GetAppSecurityKey(ctx context.Context) (string, error) {
// No authz checks
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return "", err
}
return q.db.GetAppSecurityKey(ctx)
}

Expand Down Expand Up @@ -1364,6 +1366,13 @@ func (q *querier) GetAuthorizationUserRoles(ctx context.Context, userID uuid.UUI
return q.db.GetAuthorizationUserRoles(ctx, userID)
}

func (q *querier) GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return "", err
}
return q.db.GetCoordinatorResumeTokenSigningKey(ctx)
}

func (q *querier) GetDBCryptKeys(ctx context.Context) ([]database.DBCryptKey, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
Expand Down Expand Up @@ -3788,7 +3797,9 @@ func (q *querier) UpsertAnnouncementBanners(ctx context.Context, value string) e
}

func (q *querier) UpsertAppSecurityKey(ctx context.Context, data string) error {
// No authz checks as this is done during startup
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
return err
}
return q.db.UpsertAppSecurityKey(ctx, data)
}

Expand All @@ -3799,6 +3810,13 @@ func (q *querier) UpsertApplicationName(ctx context.Context, value string) error
return q.db.UpsertApplicationName(ctx, value)
}

func (q *querier) UpsertCoordinatorResumeTokenSigningKey(ctx context.Context, value string) error {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
return err
}
return q.db.UpsertCoordinatorResumeTokenSigningKey(ctx, value)
}

func (q *querier) UpsertDefaultProxy(ctx context.Context, arg database.UpsertDefaultProxyParams) error {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
return err
Expand Down
11 changes: 9 additions & 2 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2519,10 +2519,10 @@ func (s *MethodTestSuite) TestSystemFunctions() {
check.Args(int32(0)).Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetAppSecurityKey", s.Subtest(func(db database.Store, check *expects) {
check.Args().Asserts()
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("UpsertAppSecurityKey", s.Subtest(func(db database.Store, check *expects) {
check.Args("").Asserts()
check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("GetApplicationName", s.Subtest(func(db database.Store, check *expects) {
db.UpsertApplicationName(context.Background(), "foo")
Expand Down Expand Up @@ -2562,6 +2562,13 @@ func (s *MethodTestSuite) TestSystemFunctions() {
db.UpsertOAuthSigningKey(context.Background(), "foo")
check.Args().Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("UpsertCoordinatorResumeTokenSigningKey", s.Subtest(func(db database.Store, check *expects) {
check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("GetCoordinatorResumeTokenSigningKey", s.Subtest(func(db database.Store, check *expects) {
db.UpsertCoordinatorResumeTokenSigningKey(context.Background(), "foo")
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("InsertMissingGroups", s.Subtest(func(db database.Store, check *expects) {
check.Args(database.InsertMissingGroupsParams{}).Asserts(rbac.ResourceSystem, policy.ActionCreate).Errors(errMatchAny)
}))
Expand Down
46 changes: 32 additions & 14 deletions coderd/database/dbmem/dbmem.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,20 +196,21 @@ type data struct {
customRoles []database.CustomRole
// Locks is a map of lock names. Any keys within the map are currently
// locked.
locks map[int64]struct{}
deploymentID string
derpMeshKey string
lastUpdateCheck []byte
announcementBanners []byte
healthSettings []byte
notificationsSettings []byte
applicationName string
logoURL string
appSecurityKey string
oauthSigningKey string
lastLicenseID int32
defaultProxyDisplayName string
defaultProxyIconURL string
locks map[int64]struct{}
deploymentID string
derpMeshKey string
lastUpdateCheck []byte
announcementBanners []byte
healthSettings []byte
notificationsSettings []byte
applicationName string
logoURL string
appSecurityKey string
oauthSigningKey string
coordinatorResumeTokenSigningKey string
lastLicenseID int32
defaultProxyDisplayName string
defaultProxyIconURL string
}

func validateDatabaseTypeWithValid(v reflect.Value) (handled bool, err error) {
Expand Down Expand Up @@ -2222,6 +2223,15 @@ func (q *FakeQuerier) GetAuthorizationUserRoles(_ context.Context, userID uuid.U
}, nil
}

func (q *FakeQuerier) GetCoordinatorResumeTokenSigningKey(_ context.Context) (string, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
if q.coordinatorResumeTokenSigningKey == "" {
return "", sql.ErrNoRows
}
return q.coordinatorResumeTokenSigningKey, nil
}

func (q *FakeQuerier) GetDBCryptKeys(_ context.Context) ([]database.DBCryptKey, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
Expand Down Expand Up @@ -8929,6 +8939,14 @@ func (q *FakeQuerier) UpsertApplicationName(_ context.Context, data string) erro
return nil
}

func (q *FakeQuerier) UpsertCoordinatorResumeTokenSigningKey(_ context.Context, value string) error {
q.mutex.Lock()
defer q.mutex.Unlock()

q.coordinatorResumeTokenSigningKey = value
return nil
}

func (q *FakeQuerier) UpsertDefaultProxy(_ context.Context, arg database.UpsertDefaultProxyParams) error {
q.defaultProxyDisplayName = arg.DisplayName
q.defaultProxyIconURL = arg.IconUrl
Expand Down
14 changes: 14 additions & 0 deletions coderd/database/dbmetrics/dbmetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 29 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading