Skip to content

feat: add resume support to coordinator connections #14234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 20, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 38 additions & 4 deletions cli/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -791,18 +791,52 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
}
}

keyBytes, err := hex.DecodeString(oauthSigningKeyStr)
oauthKeyBytes, err := hex.DecodeString(oauthSigningKeyStr)
if err != nil {
return xerrors.Errorf("decode oauth signing key from database: %w", err)
}
if len(keyBytes) != len(options.OAuthSigningKey) {
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(keyBytes))
if len(oauthKeyBytes) != len(options.OAuthSigningKey) {
return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(oauthKeyBytes))
}
copy(options.OAuthSigningKey[:], keyBytes)
copy(options.OAuthSigningKey[:], oauthKeyBytes)
if options.OAuthSigningKey == [32]byte{} {
return xerrors.Errorf("oauth signing key in database is empty")
}

// Read the coordinator resume token signing key from the
// database.
resumeTokenKey := [64]byte{}
resumeTokenKeyStr, err := tx.GetCoordinatorResumeTokenSigningKey(ctx)
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
return xerrors.Errorf("get coordinator resume token key: %w", err)
}
if decoded, err := hex.DecodeString(resumeTokenKeyStr); err != nil || len(decoded) != len(resumeTokenKey) {
b := make([]byte, len(resumeTokenKey))
_, err := rand.Read(b)
if err != nil {
return xerrors.Errorf("generate fresh coordinator resume token key: %w", err)
}

resumeTokenKeyStr = hex.EncodeToString(b)
err = tx.UpsertCoordinatorResumeTokenSigningKey(ctx, resumeTokenKeyStr)
if err != nil {
return xerrors.Errorf("insert freshly generated coordinator resume token key to database: %w", err)
}
}

resumeTokenKeyBytes, err := hex.DecodeString(resumeTokenKeyStr)
if err != nil {
return xerrors.Errorf("decode coordinator resume token key from database: %w", err)
}
if len(resumeTokenKeyBytes) != len(resumeTokenKey) {
return xerrors.Errorf("coordinator resume token key in database is not the correct length, expect %d got %d", len(resumeTokenKey), len(resumeTokenKeyBytes))
}
copy(resumeTokenKey[:], resumeTokenKeyBytes)
if resumeTokenKey == [64]byte{} {
return xerrors.Errorf("coordinator resume token key in database is empty")
}
options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeTokenKey, tailnet.DefaultResumeTokenExpiry)

return nil
}, nil)
if err != nil {
Expand Down
10 changes: 10 additions & 0 deletions coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,9 @@ type Options struct {
// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
// workspace applications. It consists of both a signing and encryption key.
AppSecurityKey workspaceapps.SecurityKey
// CoordinatorResumeTokenProvider is used to provide and validate resume
// tokens issued by and passed to the coordinator DRPC API.
CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider

HealthcheckFunc func(ctx context.Context, apiKey string) *healthsdk.HealthcheckReport
HealthcheckTimeout time.Duration
Expand Down Expand Up @@ -583,12 +586,16 @@ func New(options *Options) *API {
api.Options.NetworkTelemetryBatchMaxSize,
api.handleNetworkTelemetry,
)
if options.CoordinatorResumeTokenProvider == nil {
panic("CoordinatorResumeTokenProvider is nil")
}
api.TailnetClientService, err = tailnet.NewClientService(tailnet.ClientServiceOptions{
Logger: api.Logger.Named("tailnetclient"),
CoordPtr: &api.TailnetCoordinator,
DERPMapUpdateFrequency: api.Options.DERPMapUpdateFrequency,
DERPMapFn: api.DERPMap,
NetworkTelemetryHandler: api.NetworkTelemetryBatcher.Handler,
ResumeTokenProvider: api.Options.CoordinatorResumeTokenProvider,
})
if err != nil {
api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err))
Expand All @@ -613,6 +620,9 @@ func New(options *Options) *API {
options.WorkspaceAppsStatsCollectorOptions.Reporter = api.statsReporter
}

if options.AppSecurityKey.IsZero() {
api.Logger.Fatal(api.ctx, "app security key cannot be zero")
}
api.workspaceAppServer = &workspaceapps.Server{
Logger: workspaceAppsLogger,

Expand Down
1 change: 1 addition & 0 deletions coderd/coderdtest/coderdtest.go
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
TailnetCoordinator: options.Coordinator,
BaseDERPMap: derpMap,
DERPMapUpdateFrequency: 150 * time.Millisecond,
CoordinatorResumeTokenProvider: tailnet.InsecureTestResumeTokenProvider,
MetricsCacheRefreshInterval: options.MetricsCacheRefreshInterval,
AgentStatsRefreshInterval: options.AgentStatsRefreshInterval,
DeploymentValues: options.DeploymentValues,
Expand Down
22 changes: 20 additions & 2 deletions coderd/database/dbauthz/dbauthz.go
Original file line number Diff line number Diff line change
Expand Up @@ -1252,7 +1252,9 @@ func (q *querier) GetAnnouncementBanners(ctx context.Context) (string, error) {
}

func (q *querier) GetAppSecurityKey(ctx context.Context) (string, error) {
// No authz checks
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return "", err
}
return q.db.GetAppSecurityKey(ctx)
}

Expand Down Expand Up @@ -1284,6 +1286,13 @@ func (q *querier) GetAuthorizationUserRoles(ctx context.Context, userID uuid.UUI
return q.db.GetAuthorizationUserRoles(ctx, userID)
}

func (q *querier) GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
return "", err
}
return q.db.GetCoordinatorResumeTokenSigningKey(ctx)
}

func (q *querier) GetDBCryptKeys(ctx context.Context) ([]database.DBCryptKey, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
return nil, err
Expand Down Expand Up @@ -3645,7 +3654,9 @@ func (q *querier) UpsertAnnouncementBanners(ctx context.Context, value string) e
}

func (q *querier) UpsertAppSecurityKey(ctx context.Context, data string) error {
// No authz checks as this is done during startup
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
return err
}
return q.db.UpsertAppSecurityKey(ctx, data)
}

Expand All @@ -3656,6 +3667,13 @@ func (q *querier) UpsertApplicationName(ctx context.Context, value string) error
return q.db.UpsertApplicationName(ctx, value)
}

func (q *querier) UpsertCoordinatorResumeTokenSigningKey(ctx context.Context, value string) error {
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
return err
}
return q.db.UpsertCoordinatorResumeTokenSigningKey(ctx, value)
}

// UpsertCustomRole does a series of authz checks to protect custom roles.
// - Check custom roles are valid for their resource types + actions
// - Check the actor can create the custom role
Expand Down
11 changes: 9 additions & 2 deletions coderd/database/dbauthz/dbauthz_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2445,10 +2445,10 @@ func (s *MethodTestSuite) TestSystemFunctions() {
check.Args(int32(0)).Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("GetAppSecurityKey", s.Subtest(func(db database.Store, check *expects) {
check.Args().Asserts()
check.Args().Asserts(rbac.ResourceSystem, policy.ActionRead)
}))
s.Run("UpsertAppSecurityKey", s.Subtest(func(db database.Store, check *expects) {
check.Args("").Asserts()
check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("GetApplicationName", s.Subtest(func(db database.Store, check *expects) {
db.UpsertApplicationName(context.Background(), "foo")
Expand Down Expand Up @@ -2488,6 +2488,13 @@ func (s *MethodTestSuite) TestSystemFunctions() {
db.UpsertOAuthSigningKey(context.Background(), "foo")
check.Args().Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("UpsertCoordinatorResumeTokenSigningKey", s.Subtest(func(db database.Store, check *expects) {
check.Args("foo").Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("GetCoordinatorResumeTokenSigningKey", s.Subtest(func(db database.Store, check *expects) {
db.UpsertCoordinatorResumeTokenSigningKey(context.Background(), "foo")
check.Args().Asserts(rbac.ResourceSystem, policy.ActionUpdate)
}))
s.Run("InsertMissingGroups", s.Subtest(func(db database.Store, check *expects) {
check.Args(database.InsertMissingGroupsParams{}).Asserts(rbac.ResourceSystem, policy.ActionCreate).Errors(errMatchAny)
}))
Expand Down
46 changes: 32 additions & 14 deletions coderd/database/dbmem/dbmem.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,20 +196,21 @@ type data struct {
customRoles []database.CustomRole
// Locks is a map of lock names. Any keys within the map are currently
// locked.
locks map[int64]struct{}
deploymentID string
derpMeshKey string
lastUpdateCheck []byte
announcementBanners []byte
healthSettings []byte
notificationsSettings []byte
applicationName string
logoURL string
appSecurityKey string
oauthSigningKey string
lastLicenseID int32
defaultProxyDisplayName string
defaultProxyIconURL string
locks map[int64]struct{}
deploymentID string
derpMeshKey string
lastUpdateCheck []byte
announcementBanners []byte
healthSettings []byte
notificationsSettings []byte
applicationName string
logoURL string
appSecurityKey string
oauthSigningKey string
coordinatorResumeTokenSigningKey string
lastLicenseID int32
defaultProxyDisplayName string
defaultProxyIconURL string
}

func validateDatabaseTypeWithValid(v reflect.Value) (handled bool, err error) {
Expand Down Expand Up @@ -2172,6 +2173,15 @@ func (q *FakeQuerier) GetAuthorizationUserRoles(_ context.Context, userID uuid.U
}, nil
}

func (q *FakeQuerier) GetCoordinatorResumeTokenSigningKey(_ context.Context) (string, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
if q.coordinatorResumeTokenSigningKey == "" {
return "", sql.ErrNoRows
}
return q.coordinatorResumeTokenSigningKey, nil
}

func (q *FakeQuerier) GetDBCryptKeys(_ context.Context) ([]database.DBCryptKey, error) {
q.mutex.RLock()
defer q.mutex.RUnlock()
Expand Down Expand Up @@ -8800,6 +8810,14 @@ func (q *FakeQuerier) UpsertApplicationName(_ context.Context, data string) erro
return nil
}

func (q *FakeQuerier) UpsertCoordinatorResumeTokenSigningKey(_ context.Context, value string) error {
q.mutex.Lock()
defer q.mutex.Unlock()

q.coordinatorResumeTokenSigningKey = value
return nil
}

func (q *FakeQuerier) UpsertCustomRole(_ context.Context, arg database.UpsertCustomRoleParams) (database.CustomRole, error) {
err := validateDatabaseType(arg)
if err != nil {
Expand Down
14 changes: 14 additions & 0 deletions coderd/database/dbmetrics/dbmetrics.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

29 changes: 29 additions & 0 deletions coderd/database/dbmock/dbmock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions coderd/database/querier.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 21 additions & 0 deletions coderd/database/queries.sql.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions coderd/database/queries/siteconfig.sql
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@ SELECT value FROM site_configs WHERE key = 'oauth_signing_key';
INSERT INTO site_configs (key, value) VALUES ('oauth_signing_key', $1)
ON CONFLICT (key) DO UPDATE set value = $1 WHERE site_configs.key = 'oauth_signing_key';

-- name: GetCoordinatorResumeTokenSigningKey :one
SELECT value FROM site_configs WHERE key = 'coordinator_resume_token_signing_key';

-- name: UpsertCoordinatorResumeTokenSigningKey :exec
INSERT INTO site_configs (key, value) VALUES ('coordinator_resume_token_signing_key', $1)
ON CONFLICT (key) DO UPDATE set value = $1 WHERE site_configs.key = 'coordinator_resume_token_signing_key';

-- name: GetHealthSettings :one
SELECT
COALESCE((SELECT value FROM site_configs WHERE key = 'health_settings'), '{}') :: text AS health_settings
Expand Down
Loading
Loading