From b745c8eab95584fe00f7c49e57f79f2b07b6df9f Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sat, 5 Oct 2024 01:01:46 +0000 Subject: [PATCH 01/39] feat: enable key rotation --- coderd/coderd.go | 14 ++++++++++++ coderd/cryptokeys/rotate.go | 3 +++ coderd/database/dbauthz/dbauthz.go | 23 ++++++++++++++++++++ coderd/userauth.go | 34 +++++++++++++++--------------- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index cb0884808ef27..2c7d2ea502aa0 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -40,6 +40,7 @@ import ( "github.com/coder/quartz" "github.com/coder/serpent" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/entitlements" "github.com/coder/coder/v2/coderd/idpsync" "github.com/coder/coder/v2/coderd/runtimeconfig" @@ -609,6 +610,14 @@ func New(options *Options) *API { api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err)) } + // Start a background process that rotates keys. + err = cryptokeys.StartRotator(api.ctx, api.Logger.Named("keyrotator"), api.Database) + if err != nil { + api.Logger.Fatal(api.ctx, "start key rotator", slog.Error(err)) + } + + api.oauthConvertKeycache = cryptokeys.NewDBCache(api.Logger.Named("oauth_convert_keycache"), api.Database, database.CryptoKeyFeatureOidcConvert) + api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ Database: options.Database, Logger: options.Logger.Named("workspacestats"), @@ -1389,6 +1398,11 @@ type API struct { // dbRolluper rolls up template usage stats from raw agent and app // stats. This is used to provide insights in the WebUI. dbRolluper *dbrollup.Rolluper + + // resumeTokenKeycache is used to fetch and cache keys used for signing JWTs + // oauthConvertKeycache is used to fetch and cache keys used for signing JWTs + // during OAuth conversions. See userauth.go.convertUserToOauth. + oauthConvertKeycache cryptokeys.Keycache } // Close waits for all WebSocket connections to drain before returning. diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index 14a623e2156db..f21375a3ff71e 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -11,6 +11,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/quartz" ) @@ -54,6 +55,8 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption { // It ensures there's at least one valid key per feature prior to returning. // Canceling the provided context will stop the background process. func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, opts ...RotatorOption) error { + //nolint:gocritic // KeyRotator can only rotate crypto keys. + ctx = dbauthz.AsSystemRestricted(ctx) kr := &rotator{ db: db, logger: logger, diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index a587788791c35..f1eb43d2a4ffb 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -228,6 +228,24 @@ var ( Scope: rbac.ScopeAll, }.WithCachedASTValue() + // See cryptokeys package. + subjectCryptoKey = rbac.Subject{ + FriendlyName: "Crypto Key Rotator", + ID: uuid.Nil.String(), + Roles: rbac.Roles([]rbac.Role{ + { + Identifier: rbac.RoleIdentifier{Name: "keyrotator"}, + DisplayName: "Key Rotator", + Site: rbac.Permissions(map[string][]policy.Action{ + rbac.ResourceCryptoKey.Type: {policy.WildcardSymbol, policy.ActionRead}, + }), + Org: map[string][]rbac.Permission{}, + User: []rbac.Permission{}, + }, + }), + Scope: rbac.ScopeAll, + }.WithCachedASTValue() + subjectSystemRestricted = rbac.Subject{ FriendlyName: "System", ID: uuid.Nil.String(), @@ -281,6 +299,11 @@ func AsHangDetector(ctx context.Context) context.Context { return context.WithValue(ctx, authContextKey{}, subjectHangDetector) } +// AsKeyRotator returns a context with an actor that has permissions required for rotating crypto keys. +func AsKeyRotator(ctx context.Context) context.Context { + return context.WithValue(ctx, authContextKey{}, subjectCryptoKey) +} + // AsSystemRestricted returns a context with an actor that has permissions // required for various system operations (login, logout, metrics cache). func AsSystemRestricted(ctx context.Context) context.Context { diff --git a/coderd/userauth.go b/coderd/userauth.go index 0ff3dfa8f97cc..bb2af521b7864 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -15,7 +15,8 @@ import ( "time" "github.com/coreos/go-oidc/v3/oidc" - "github.com/golang-jwt/jwt/v4" + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/go-github/v43/github" "github.com/google/uuid" "github.com/moby/moby/pkg/namesgenerator" @@ -23,6 +24,9 @@ import ( "golang.org/x/xerrors" "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/coderd/idpsync" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/apikey" "github.com/coder/coder/v2/coderd/audit" @@ -49,7 +53,7 @@ const ( ) type OAuthConvertStateClaims struct { - jwt.RegisteredClaims + jwt.Claims UserID uuid.UUID `json:"user_id"` State string `json:"state"` @@ -149,11 +153,11 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { // Eg: Developers with more than 1 deployment. now := time.Now() claims := &OAuthConvertStateClaims{ - RegisteredClaims: jwt.RegisteredClaims{ + Claims: jwt.Claims{ Issuer: api.DeploymentID, Subject: stateString, Audience: []string{user.ID.String()}, - ExpiresAt: jwt.NewNumericDate(now.Add(time.Minute * 5)), + Expiry: jwt.NewNumericDate(now.Add(time.Minute * 5)), NotBefore: jwt.NewNumericDate(now.Add(time.Second * -1)), IssuedAt: jwt.NewNumericDate(now), ID: uuid.NewString(), @@ -164,9 +168,7 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { ToLoginType: req.ToType, } - token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims) - // Key must be a byte slice, not an array. So make sure to include the [:] - tokenString, err := token.SignedString(api.OAuthSigningKey[:]) + token, err := jwtutils.Sign(dbauthz.AsKeyRotator(ctx), api.oauthConvertKeycache, claims) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error signing state jwt.", @@ -176,8 +178,8 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { } aReq.New = database.AuditOAuthConvertState{ - CreatedAt: claims.IssuedAt.Time, - ExpiresAt: claims.ExpiresAt.Time, + CreatedAt: claims.IssuedAt.Time(), + ExpiresAt: claims.Expiry.Time(), FromLoginType: database.LoginType(claims.FromLoginType), ToLoginType: database.LoginType(claims.ToLoginType), UserID: claims.UserID, @@ -186,8 +188,8 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { http.SetCookie(rw, &http.Cookie{ Name: OAuthConvertCookieValue, Path: "/", - Value: tokenString, - Expires: claims.ExpiresAt.Time, + Value: token, + Expires: claims.Expiry.Time(), Secure: api.SecureAuthCookie, HttpOnly: true, // Must be SameSite to work on the redirected auth flow from the @@ -196,7 +198,7 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { }) httpapi.Write(ctx, rw, http.StatusCreated, codersdk.OAuthConversionResponse{ StateString: stateString, - ExpiresAt: claims.ExpiresAt.Time, + ExpiresAt: claims.Expiry.Time(), ToType: claims.ToLoginType, UserID: claims.UserID, }) @@ -1675,10 +1677,8 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data } } var claims OAuthConvertStateClaims - token, err := jwt.ParseWithClaims(jwtCookie.Value, &claims, func(_ *jwt.Token) (interface{}, error) { - return api.OAuthSigningKey[:], nil - }) - if xerrors.Is(err, jwt.ErrSignatureInvalid) || !token.Valid { + err = jwtutils.Verify(dbauthz.AsKeyRotator(ctx), api.oauthConvertKeycache, jwtCookie.Value, &claims) + if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) { // These errors are probably because the user is mixing 2 coder deployments. return database.User{}, idpsync.HTTPError{ Code: http.StatusBadRequest, @@ -1707,7 +1707,7 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data oauthConvertAudit.UserID = claims.UserID oauthConvertAudit.Old = user - if claims.RegisteredClaims.Issuer != api.DeploymentID { + if claims.Issuer != api.DeploymentID { return database.User{}, idpsync.HTTPError{ Code: http.StatusForbidden, Msg: "Request to convert login type failed. Issuer mismatch. Found a cookie from another coder deployment, please try again.", From b98bff00a0fcbbe026172ca2cf5df2e061b5407f Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sat, 5 Oct 2024 02:47:32 +0000 Subject: [PATCH 02/39] add migration --- .../000262_site_to_crypto_key.down.sql | 0 .../000262_site_to_crypto_key.u.sql | 52 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 coderd/database/migrations/000262_site_to_crypto_key.down.sql create mode 100644 coderd/database/migrations/000262_site_to_crypto_key.u.sql diff --git a/coderd/database/migrations/000262_site_to_crypto_key.down.sql b/coderd/database/migrations/000262_site_to_crypto_key.down.sql new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/coderd/database/migrations/000262_site_to_crypto_key.u.sql b/coderd/database/migrations/000262_site_to_crypto_key.u.sql new file mode 100644 index 0000000000000..3ca41ce95f76d --- /dev/null +++ b/coderd/database/migrations/000262_site_to_crypto_key.u.sql @@ -0,0 +1,52 @@ +-- Create a new enum type with the desired values +CREATE TYPE new_crypto_key_feature AS ENUM ( + 'workspace_apps_token', + 'workspace_apps_api_key', + 'oidc_convert', + 'tailnet_resume' +); + +-- Drop the old type and rename the new one +ALTER TABLE crypto_keys + ALTER COLUMN feature TYPE new_crypto_key_feature + USING (feature::text::new_crypto_key_feature); + +DROP TYPE crypto_key_feature; + +ALTER TYPE new_crypto_key_feature RENAME TO crypto_key_feature; + +-- Extract and decode the app_signing_key, then insert the first 64 bytes for workspace_apps_token +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +SELECT + 'workspace_apps_token'::crypto_key_feature, + 1, + encode(substring(decode(value, 'hex') from 1 for 64), 'base64'), + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + NULL +FROM site_configs +WHERE key = 'app_signing_key'; + +-- Extract and decode the app_signing_key, then insert the last 32 bytes for workspace_apps_api_key +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +SELECT + 'workspace_apps_api_key'::crypto_key_feature, + 1, + encode(substring(decode(value, 'hex') from -32), 'base64'), + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + NULL +FROM site_configs +WHERE key = 'app_signing_key'; + +-- Extract and decode the coordinator_resume_token_signing_key, then insert it for tailnet_resume feature +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +SELECT + 'tailnet_resume'::crypto_key_feature, + 1, + encode(decode(value, 'hex'), 'base64'), + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + NULL +FROM site_configs +WHERE key = 'coordinator_resume_token_signing_key'; From 0646b306309e30645ef482b10e8fb148890e92c3 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sat, 5 Oct 2024 03:28:37 +0000 Subject: [PATCH 03/39] Refactor cryptographic key handling for OIDC and API keys This commit updates the cryptographic key handling by separating workspace app token and API key features. It corrects feature identifiers for clearer distinction between OIDC conversion and API key usage, enhancing the maintainability and clarity around cryptographic key usage within the system. Additionally, reworks related tests and migration scripts to align with these changes. --- coderd/coderd.go | 7 +- coderd/cryptokeys/rotate.go | 8 +-- coderd/cryptokeys/rotate_internal_test.go | 64 +++++++++---------- coderd/cryptokeys/rotate_test.go | 4 +- coderd/database/dbauthz/dbauthz_test.go | 14 ++-- coderd/database/dbgen/dbgen.go | 6 +- coderd/database/dump.sql | 3 +- .../000262_site_to_crypto_key.down.sql | 19 ++++++ ...u.sql => 000262_site_to_crypto_key.up.sql} | 0 .../fixtures/000262_site_to_crypto_key.up.sql | 15 +++++ coderd/database/models.go | 17 +++-- coderd/database/sqlc.yaml | 2 + coderd/jwtutils/jwt_test.go | 4 +- coderd/userauth.go | 1 - enterprise/coderd/workspaceproxy.go | 2 +- enterprise/coderd/workspaceproxy_test.go | 8 +-- enterprise/dbcrypt/dbcrypt_internal_test.go | 10 +-- 17 files changed, 113 insertions(+), 71 deletions(-) rename coderd/database/migrations/{000262_site_to_crypto_key.u.sql => 000262_site_to_crypto_key.up.sql} (100%) create mode 100644 coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql diff --git a/coderd/coderd.go b/coderd/coderd.go index 2c7d2ea502aa0..c8577219c7b15 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -616,7 +616,10 @@ func New(options *Options) *API { api.Logger.Fatal(api.ctx, "start key rotator", slog.Error(err)) } - api.oauthConvertKeycache = cryptokeys.NewDBCache(api.Logger.Named("oauth_convert_keycache"), api.Database, database.CryptoKeyFeatureOidcConvert) + api.oauthConvertKeycache, err = cryptokeys.NewSigningCache(api.Logger.Named("oauth_convert_keycache"), api.Database, database.CryptoKeyFeatureOIDCConvert) + if err != nil { + api.Logger.Fatal(api.ctx, "failed to initialize oauth convert key cache", slog.Error(err)) + } api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ Database: options.Database, @@ -1402,7 +1405,7 @@ type API struct { // resumeTokenKeycache is used to fetch and cache keys used for signing JWTs // oauthConvertKeycache is used to fetch and cache keys used for signing JWTs // during OAuth conversions. See userauth.go.convertUserToOauth. - oauthConvertKeycache cryptokeys.Keycache + oauthConvertKeycache cryptokeys.SigningKeycache } // Close waits for all WebSocket connections to drain before returning. diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index f21375a3ff71e..f5e36eca1da6c 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -229,9 +229,9 @@ func (k *rotator) rotateKey(ctx context.Context, tx database.Store, key database func generateNewSecret(feature database.CryptoKeyFeature) (string, error) { switch feature { - case database.CryptoKeyFeatureWorkspaceApps: + case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return generateKey(32) - case database.CryptoKeyFeatureOidcConvert: + case database.CryptoKeyFeatureOIDCConvert: return generateKey(64) case database.CryptoKeyFeatureTailnetResume: return generateKey(64) @@ -250,9 +250,9 @@ func generateKey(length int) (string, error) { func tokenDuration(feature database.CryptoKeyFeature) time.Duration { switch feature { - case database.CryptoKeyFeatureWorkspaceApps: + case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return WorkspaceAppsTokenDuration - case database.CryptoKeyFeatureOidcConvert: + case database.CryptoKeyFeatureOIDCConvert: return OIDCConvertTokenDuration case database.CryptoKeyFeatureTailnetResume: return TailnetResumeTokenDuration diff --git a/coderd/cryptokeys/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go index 43754c1d8750f..ced266477746a 100644 --- a/coderd/cryptokeys/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -38,7 +38,7 @@ func Test_rotateKeys(t *testing.T) { clock: clock, logger: logger, features: []database.CryptoKeyFeature{ - database.CryptoKeyFeatureWorkspaceApps, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, }, } @@ -46,7 +46,7 @@ func Test_rotateKeys(t *testing.T) { // Seed the database with an existing key. oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now, Sequence: 15, }) @@ -69,11 +69,11 @@ func Test_rotateKeys(t *testing.T) { // The new key should be created and have a starts_at of the old key's expires_at. newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: oldKey.Sequence + 1, }) require.NoError(t, err) - requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceApps, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1) + requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceAppsAPIKey, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1) // Advance the clock just before the keys delete time. clock.Advance(oldKey.DeletesAt.Time.UTC().Sub(now) - time.Second) @@ -123,7 +123,7 @@ func Test_rotateKeys(t *testing.T) { clock: clock, logger: logger, features: []database.CryptoKeyFeature{ - database.CryptoKeyFeatureWorkspaceApps, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, }, } @@ -131,7 +131,7 @@ func Test_rotateKeys(t *testing.T) { // Seed the database with an existing key existingKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now, Sequence: 123, }) @@ -179,7 +179,7 @@ func Test_rotateKeys(t *testing.T) { clock: clock, logger: logger, features: []database.CryptoKeyFeature{ - database.CryptoKeyFeatureWorkspaceApps, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, }, } @@ -187,7 +187,7 @@ func Test_rotateKeys(t *testing.T) { // Seed the database with an existing key deletingKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-keyDuration), Sequence: 789, DeletesAt: sql.NullTime{ @@ -232,7 +232,7 @@ func Test_rotateKeys(t *testing.T) { clock: clock, logger: logger, features: []database.CryptoKeyFeature{ - database.CryptoKeyFeatureWorkspaceApps, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, }, } @@ -240,7 +240,7 @@ func Test_rotateKeys(t *testing.T) { // Seed the database with an existing key deletingKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now, Sequence: 456, DeletesAt: sql.NullTime{ @@ -281,7 +281,7 @@ func Test_rotateKeys(t *testing.T) { clock: clock, logger: logger, features: []database.CryptoKeyFeature{ - database.CryptoKeyFeatureWorkspaceApps, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, }, } @@ -291,7 +291,7 @@ func Test_rotateKeys(t *testing.T) { keys, err := db.GetCryptoKeys(ctx) require.NoError(t, err) require.Len(t, keys, 1) - requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceApps, clock.Now().UTC(), nullTime, 1) + requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceAppsAPIKey, clock.Now().UTC(), nullTime, 1) }) // Assert we insert a new key when the only key was manually deleted. @@ -312,14 +312,14 @@ func Test_rotateKeys(t *testing.T) { clock: clock, logger: logger, features: []database.CryptoKeyFeature{ - database.CryptoKeyFeatureWorkspaceApps, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, }, } now := dbnow(clock) deletedkey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now, Sequence: 19, DeletesAt: sql.NullTime{ @@ -338,7 +338,7 @@ func Test_rotateKeys(t *testing.T) { keys, err := db.GetCryptoKeys(ctx) require.NoError(t, err) require.Len(t, keys, 1) - requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceApps, now, nullTime, deletedkey.Sequence+1) + requireKey(t, keys[0], database.CryptoKeyFeatureWorkspaceAppsAPIKey, now, nullTime, deletedkey.Sequence+1) }) // This tests ensures that rotation works with multiple @@ -389,14 +389,14 @@ func Test_rotateKeys(t *testing.T) { // Insert a key that should be rotated. rotatedKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-keyDuration + time.Hour), Sequence: 42, }) // Insert a key that should not trigger an action. validKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureOidcConvert, + Feature: database.CryptoKeyFeatureOIDCConvert, StartsAt: now, Sequence: 17, }) @@ -412,20 +412,20 @@ func Test_rotateKeys(t *testing.T) { require.NoError(t, err) // No actions on OIDC convert. - require.Len(t, kbf[database.CryptoKeyFeatureOidcConvert], 1) + require.Len(t, kbf[database.CryptoKeyFeatureOIDCConvert], 1) // Workspace apps should have been rotated. - require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceApps], 2) + require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey], 2) // No existing key for tailnet resume should've // caused a key to be inserted. require.Len(t, kbf[database.CryptoKeyFeatureTailnetResume], 1) - oidcKey := kbf[database.CryptoKeyFeatureOidcConvert][0] + oidcKey := kbf[database.CryptoKeyFeatureOIDCConvert][0] tailnetKey := kbf[database.CryptoKeyFeatureTailnetResume][0] - requireKey(t, oidcKey, database.CryptoKeyFeatureOidcConvert, now, nullTime, validKey.Sequence) + requireKey(t, oidcKey, database.CryptoKeyFeatureOIDCConvert, now, nullTime, validKey.Sequence) requireKey(t, tailnetKey, database.CryptoKeyFeatureTailnetResume, now, nullTime, deletedKey.Sequence+1) - newKey := kbf[database.CryptoKeyFeatureWorkspaceApps][0] - oldKey := kbf[database.CryptoKeyFeatureWorkspaceApps][1] + newKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][0] + oldKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][1] if newKey.Sequence == rotatedKey.Sequence { oldKey, newKey = newKey, oldKey } @@ -433,8 +433,8 @@ func Test_rotateKeys(t *testing.T) { Time: rotatedKey.ExpiresAt(keyDuration).Add(WorkspaceAppsTokenDuration + time.Hour), Valid: true, } - requireKey(t, oldKey, database.CryptoKeyFeatureWorkspaceApps, rotatedKey.StartsAt.UTC(), deletesAt, rotatedKey.Sequence) - requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceApps, rotatedKey.ExpiresAt(keyDuration), nullTime, rotatedKey.Sequence+1) + requireKey(t, oldKey, database.CryptoKeyFeatureWorkspaceAppsAPIKey, rotatedKey.StartsAt.UTC(), deletesAt, rotatedKey.Sequence) + requireKey(t, newKey, database.CryptoKeyFeatureWorkspaceAppsAPIKey, rotatedKey.ExpiresAt(keyDuration), nullTime, rotatedKey.Sequence+1) }) t.Run("UnknownFeature", func(t *testing.T) { @@ -478,11 +478,11 @@ func Test_rotateKeys(t *testing.T) { keyDuration: keyDuration, clock: clock, logger: logger, - features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceApps}, + features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceAppsAPIKey}, } expiringKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-keyDuration), Sequence: 345, }) @@ -522,19 +522,19 @@ func Test_rotateKeys(t *testing.T) { keyDuration: keyDuration, clock: clock, logger: logger, - features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceApps}, + features: []database.CryptoKeyFeature{database.CryptoKeyFeatureWorkspaceAppsAPIKey}, } now := dbnow(clock) expiredKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-keyDuration - 2*time.Hour), Sequence: 19, }) deletedKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now, Sequence: 20, Secret: sql.NullString{ @@ -587,9 +587,9 @@ func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKey require.NoError(t, err) switch key.Feature { - case database.CryptoKeyFeatureOidcConvert: + case database.CryptoKeyFeatureOIDCConvert: require.Len(t, secret, 64) - case database.CryptoKeyFeatureWorkspaceApps: + case database.CryptoKeyFeatureWorkspaceAppsAPIKey: require.Len(t, secret, 32) case database.CryptoKeyFeatureTailnetResume: require.Len(t, secret, 64) diff --git a/coderd/cryptokeys/rotate_test.go b/coderd/cryptokeys/rotate_test.go index 190ad213b1153..96acacf1d0182 100644 --- a/coderd/cryptokeys/rotate_test.go +++ b/coderd/cryptokeys/rotate_test.go @@ -58,7 +58,7 @@ func TestRotator(t *testing.T) { now := clock.Now().UTC() rotatingKey := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-cryptokeys.DefaultKeyDuration + time.Hour + time.Minute), Sequence: 12345, }) @@ -85,7 +85,7 @@ func TestRotator(t *testing.T) { require.NoError(t, err) require.Len(t, keys, initialKeyLen+1) - newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps) + newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) require.Equal(t, rotatingKey.Sequence+1, newKey.Sequence) require.Equal(t, rotatingKey.ExpiresAt(cryptokeys.DefaultKeyDuration), newKey.StartsAt.UTC()) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index f554d709ad4d0..5978d64c841ad 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -2246,13 +2246,13 @@ func (s *MethodTestSuite) TestCryptoKeys() { })) s.Run("InsertCryptoKey", s.Subtest(func(db database.Store, check *expects) { check.Args(database.InsertCryptoKeyParams{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, }). Asserts(rbac.ResourceCryptoKey, policy.ActionCreate) })) s.Run("DeleteCryptoKey", s.Subtest(func(db database.Store, check *expects) { key := dbgen.CryptoKey(s.T(), db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: 4, }) check.Args(database.DeleteCryptoKeyParams{ @@ -2262,7 +2262,7 @@ func (s *MethodTestSuite) TestCryptoKeys() { })) s.Run("GetCryptoKeyByFeatureAndSequence", s.Subtest(func(db database.Store, check *expects) { key := dbgen.CryptoKey(s.T(), db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: 4, }) check.Args(database.GetCryptoKeyByFeatureAndSequenceParams{ @@ -2272,14 +2272,14 @@ func (s *MethodTestSuite) TestCryptoKeys() { })) s.Run("GetLatestCryptoKeyByFeature", s.Subtest(func(db database.Store, check *expects) { dbgen.CryptoKey(s.T(), db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: 4, }) - check.Args(database.CryptoKeyFeatureWorkspaceApps).Asserts(rbac.ResourceCryptoKey, policy.ActionRead) + check.Args(database.CryptoKeyFeatureWorkspaceAppsAPIKey).Asserts(rbac.ResourceCryptoKey, policy.ActionRead) })) s.Run("UpdateCryptoKeyDeletesAt", s.Subtest(func(db database.Store, check *expects) { key := dbgen.CryptoKey(s.T(), db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: 4, }) check.Args(database.UpdateCryptoKeyDeletesAtParams{ @@ -2289,7 +2289,7 @@ func (s *MethodTestSuite) TestCryptoKeys() { }).Asserts(rbac.ResourceCryptoKey, policy.ActionUpdate) })) s.Run("GetCryptoKeysByFeature", s.Subtest(func(db database.Store, check *expects) { - check.Args(database.CryptoKeyFeatureWorkspaceApps). + check.Args(database.CryptoKeyFeatureWorkspaceAppsAPIKey). Asserts(rbac.ResourceCryptoKey, policy.ActionRead) })) } diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index a8ecabe752011..7c28e8fdec371 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -943,7 +943,7 @@ func CustomRole(t testing.TB, db database.Store, seed database.CustomRole) datab func CryptoKey(t testing.TB, db database.Store, seed database.CryptoKey) database.CryptoKey { t.Helper() - seed.Feature = takeFirst(seed.Feature, database.CryptoKeyFeatureWorkspaceApps) + seed.Feature = takeFirst(seed.Feature, database.CryptoKeyFeatureWorkspaceAppsAPIKey) // An empty string for the secret is interpreted as // a caller wanting a new secret to be generated. @@ -1048,9 +1048,9 @@ func takeFirst[Value comparable](values ...Value) Value { func newCryptoKeySecret(feature database.CryptoKeyFeature) (string, error) { switch feature { - case database.CryptoKeyFeatureWorkspaceApps: + case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return generateCryptoKey(32) - case database.CryptoKeyFeatureOidcConvert: + case database.CryptoKeyFeatureOIDCConvert: return generateCryptoKey(64) case database.CryptoKeyFeatureTailnetResume: return generateCryptoKey(64) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 626d00cc81b41..e69f08aedb31c 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -37,7 +37,8 @@ CREATE TYPE build_reason AS ENUM ( ); CREATE TYPE crypto_key_feature AS ENUM ( - 'workspace_apps', + 'workspace_apps_token', + 'workspace_apps_api_key', 'oidc_convert', 'tailnet_resume' ); diff --git a/coderd/database/migrations/000262_site_to_crypto_key.down.sql b/coderd/database/migrations/000262_site_to_crypto_key.down.sql index e69de29bb2d1d..062b085cc2ea5 100644 --- a/coderd/database/migrations/000262_site_to_crypto_key.down.sql +++ b/coderd/database/migrations/000262_site_to_crypto_key.down.sql @@ -0,0 +1,19 @@ +-- Step 1: Remove the new entries from crypto_keys table +DELETE FROM crypto_keys +WHERE feature IN ('workspace_apps_token', 'workspace_apps_api_key', 'tailnet_resume') + AND sequence = 1; + +CREATE TYPE crypto_key_feature_old AS ENUM ( + 'workspace_apps', + 'oidc_convert', + 'tailnet_resume' +); + +ALTER TABLE crypto_keys + ALTER COLUMN feature TYPE crypto_key_feature_old + USING (feature::text::crypto_key_feature_old); + +DROP TYPE crypto_key_feature; + +ALTER TYPE crypto_key_feature_old RENAME TO crypto_key_feature; + diff --git a/coderd/database/migrations/000262_site_to_crypto_key.u.sql b/coderd/database/migrations/000262_site_to_crypto_key.up.sql similarity index 100% rename from coderd/database/migrations/000262_site_to_crypto_key.u.sql rename to coderd/database/migrations/000262_site_to_crypto_key.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql b/coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql new file mode 100644 index 0000000000000..f4e93636698da --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql @@ -0,0 +1,15 @@ +INSERT INTO site_configs (key, value) +VALUES ( + 'app_signing_key', + encode(gen_random_bytes(96), 'hex') +) +ON CONFLICT (key) DO UPDATE +SET value = EXCLUDED.value; + +INSERT INTO site_configs (key, value) +VALUES ( + 'coordinator_resume_token_signing_key', + encode(gen_random_bytes(32), 'hex') +) +ON CONFLICT (key) DO UPDATE +SET value = EXCLUDED.value; diff --git a/coderd/database/models.go b/coderd/database/models.go index 05b4c404ea16f..bd5e37bfee55c 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -342,9 +342,10 @@ func AllBuildReasonValues() []BuildReason { type CryptoKeyFeature string const ( - CryptoKeyFeatureWorkspaceApps CryptoKeyFeature = "workspace_apps" - CryptoKeyFeatureOidcConvert CryptoKeyFeature = "oidc_convert" - CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" + CryptoKeyFeatureWorkspaceAppsToken CryptoKeyFeature = "workspace_apps_token" + CryptoKeyFeatureWorkspaceAppsAPIKey CryptoKeyFeature = "workspace_apps_api_key" + CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" + CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" ) func (e *CryptoKeyFeature) Scan(src interface{}) error { @@ -384,8 +385,9 @@ func (ns NullCryptoKeyFeature) Value() (driver.Value, error) { func (e CryptoKeyFeature) Valid() bool { switch e { - case CryptoKeyFeatureWorkspaceApps, - CryptoKeyFeatureOidcConvert, + case CryptoKeyFeatureWorkspaceAppsToken, + CryptoKeyFeatureWorkspaceAppsAPIKey, + CryptoKeyFeatureOIDCConvert, CryptoKeyFeatureTailnetResume: return true } @@ -394,8 +396,9 @@ func (e CryptoKeyFeature) Valid() bool { func AllCryptoKeyFeatureValues() []CryptoKeyFeature { return []CryptoKeyFeature{ - CryptoKeyFeatureWorkspaceApps, - CryptoKeyFeatureOidcConvert, + CryptoKeyFeatureWorkspaceAppsToken, + CryptoKeyFeatureWorkspaceAppsAPIKey, + CryptoKeyFeatureOIDCConvert, CryptoKeyFeatureTailnetResume, } } diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 7ef860e0b36ce..64ee025c0ef0a 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -133,6 +133,8 @@ sql: api_key_id: APIKeyID callback_url: CallbackURL login_type_oauth2_provider_app: LoginTypeOAuth2ProviderApp + crypto_key_feature_workspace_apps_api_key: CryptoKeyFeatureWorkspaceAppsAPIKey + crypto_key_feature_oidc_convert: CryptoKeyFeatureOIDCConvert rules: - name: do-not-use-public-schema-in-queries message: "do not use public schema in queries" diff --git a/coderd/jwtutils/jwt_test.go b/coderd/jwtutils/jwt_test.go index 697e5d210d858..4c57e0451fcb6 100644 --- a/coderd/jwtutils/jwt_test.go +++ b/coderd/jwtutils/jwt_test.go @@ -236,7 +236,7 @@ func TestJWS(t *testing.T) { ctx = testutil.Context(t, testutil.WaitShort) db, _ = dbtestutil.NewDB(t) _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureOidcConvert, + Feature: database.CryptoKeyFeatureOIDCConvert, StartsAt: time.Now(), }) log = slogtest.Make(t, nil) @@ -326,7 +326,7 @@ func TestJWE(t *testing.T) { ctx = testutil.Context(t, testutil.WaitShort) db, _ = dbtestutil.NewDB(t) _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: time.Now(), }) log = slogtest.Make(t, nil) diff --git a/coderd/userauth.go b/coderd/userauth.go index bb2af521b7864..25e9b5bd88a0e 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -36,7 +36,6 @@ import ( "github.com/coder/coder/v2/coderd/externalauth" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" - "github.com/coder/coder/v2/coderd/idpsync" "github.com/coder/coder/v2/coderd/notifications" "github.com/coder/coder/v2/coderd/promoauth" "github.com/coder/coder/v2/coderd/rbac" diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index 47bdf53493489..2d07e2ef16848 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -727,7 +727,7 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request) func (api *API) workspaceProxyCryptoKeys(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() - keys, err := api.Database.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps) + keys, err := api.Database.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceAppsAPIKey) if err != nil { httpapi.InternalServerError(rw, err) return diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 5231a0b0c4241..ee871c175565d 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -915,14 +915,14 @@ func TestGetCryptoKeys(t *testing.T) { now := time.Now() expectedKey1 := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-time.Hour), Sequence: 2, }) key1 := db2sdk.CryptoKey(expectedKey1) expectedKey2 := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now, Sequence: 3, }) @@ -930,7 +930,7 @@ func TestGetCryptoKeys(t *testing.T) { // Create a deleted key. _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, StartsAt: now.Add(-time.Hour), Secret: sql.NullString{ String: "secret1", @@ -946,7 +946,7 @@ func TestGetCryptoKeys(t *testing.T) { Sequence: 1, }) _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureOidcConvert, + Feature: database.CryptoKeyFeatureOIDCConvert, StartsAt: now.Add(-time.Hour), Sequence: 1, }) diff --git a/enterprise/dbcrypt/dbcrypt_internal_test.go b/enterprise/dbcrypt/dbcrypt_internal_test.go index 432dc90061677..a96c32aaa8aae 100644 --- a/enterprise/dbcrypt/dbcrypt_internal_test.go +++ b/enterprise/dbcrypt/dbcrypt_internal_test.go @@ -397,12 +397,12 @@ func TestCryptoKeys(t *testing.T) { _ = dbgen.CryptoKey(t, crypt, database.CryptoKey{ Secret: sql.NullString{String: "test", Valid: true}, }) - key, err := crypt.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps) + key, err := crypt.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) require.Equal(t, "test", key.Secret.String) require.Equal(t, ciphers[0].HexDigest(), key.SecretKeyID.String) - key, err = db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceApps) + key, err = db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) requireEncryptedEquals(t, ciphers[0], key.Secret.String, "test") require.Equal(t, ciphers[0].HexDigest(), key.SecretKeyID.String) @@ -415,7 +415,7 @@ func TestCryptoKeys(t *testing.T) { Secret: sql.NullString{String: "test", Valid: true}, }) key, err := crypt.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: key.Sequence, }) require.NoError(t, err) @@ -423,7 +423,7 @@ func TestCryptoKeys(t *testing.T) { require.Equal(t, ciphers[0].HexDigest(), key.SecretKeyID.String) key, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: key.Sequence, }) require.NoError(t, err) @@ -459,7 +459,7 @@ func TestCryptoKeys(t *testing.T) { Secret: sql.NullString{String: "test", Valid: true}, }) _ = dbgen.CryptoKey(t, crypt, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceApps, + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, Sequence: 43, }) keys, err := crypt.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureTailnetResume) From b73b210c3edaecd00985e81a0ed42659b86480d8 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sat, 5 Oct 2024 05:16:04 +0000 Subject: [PATCH 04/39] the end is nigh --- cli/server.go | 90 +------- coderd/coderd.go | 26 ++- coderd/coderdtest/coderdtest.go | 5 - coderd/jwtutils/jwe.go | 38 ++++ coderd/jwtutils/jws.go | 27 +++ coderd/workspaceagents.go | 2 +- coderd/workspaceagents_test.go | 15 +- coderd/workspaceapps.go | 4 +- coderd/workspaceapps/db.go | 18 +- coderd/workspaceapps/db_test.go | 17 +- coderd/workspaceapps/provider.go | 4 +- coderd/workspaceapps/proxy.go | 15 +- coderd/workspaceapps/request.go | 2 +- coderd/workspaceapps/request_test.go | 2 +- coderd/workspaceapps/token.go | 203 ++---------------- coderd/workspaceapps/token_test.go | 202 +++++------------ .../workspacesdk/connector_internal_test.go | 21 +- enterprise/coderd/workspaceproxy.go | 1 - enterprise/wsproxy/tokenprovider.go | 15 +- tailnet/resume.go | 80 ++----- tailnet/resume_test.go | 36 ++-- tailnet/service.go | 2 +- 22 files changed, 273 insertions(+), 552 deletions(-) diff --git a/cli/server.go b/cli/server.go index 5adb44c3c0a7d..10e105b6ee330 100644 --- a/cli/server.go +++ b/cli/server.go @@ -10,7 +10,6 @@ import ( "crypto/tls" "crypto/x509" "database/sql" - "encoding/hex" "errors" "flag" "fmt" @@ -62,6 +61,7 @@ import ( "github.com/coder/serpent" "github.com/coder/wgtunnel/tunnelsdk" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/entitlements" "github.com/coder/coder/v2/coderd/notifications/reports" "github.com/coder/coder/v2/coderd/runtimeconfig" @@ -97,7 +97,6 @@ import ( "github.com/coder/coder/v2/coderd/updatecheck" "github.com/coder/coder/v2/coderd/util/slice" stringutil "github.com/coder/coder/v2/coderd/util/strings" - "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/coderd/workspaceapps/appurl" "github.com/coder/coder/v2/coderd/workspacestats" "github.com/coder/coder/v2/codersdk" @@ -741,90 +740,19 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("set deployment id: %w", err) } } - - // Read the app signing key from the DB. We store it hex encoded - // since the config table uses strings for the value and we - // don't want to deal with automatic encoding issues. - appSecurityKeyStr, err := tx.GetAppSecurityKey(ctx) - if err != nil && !xerrors.Is(err, sql.ErrNoRows) { - return xerrors.Errorf("get app signing key: %w", err) - } - // If the string in the DB is an invalid hex string or the - // length is not equal to the current key length, generate a new - // one. - // - // If the key is regenerated, old signed tokens and encrypted - // strings will become invalid. New signed app tokens will be - // generated automatically on failure. Any workspace app token - // smuggling operations in progress may fail, although with a - // helpful error. - if decoded, err := hex.DecodeString(appSecurityKeyStr); err != nil || len(decoded) != len(workspaceapps.SecurityKey{}) { - b := make([]byte, len(workspaceapps.SecurityKey{})) - _, err := rand.Read(b) - if err != nil { - return xerrors.Errorf("generate fresh app signing key: %w", err) - } - - appSecurityKeyStr = hex.EncodeToString(b) - err = tx.UpsertAppSecurityKey(ctx, appSecurityKeyStr) - if err != nil { - return xerrors.Errorf("insert freshly generated app signing key to database: %w", err) - } - } - - appSecurityKey, err := workspaceapps.KeyFromString(appSecurityKeyStr) - if err != nil { - return xerrors.Errorf("decode app signing key from database: %w", err) - } - - options.AppSecurityKey = appSecurityKey - - // Read the oauth signing key from the database. Like the app security, generate a new one - // if it is invalid for any reason. - oauthSigningKeyStr, err := tx.GetOAuthSigningKey(ctx) - if err != nil && !xerrors.Is(err, sql.ErrNoRows) { - return xerrors.Errorf("get app oauth signing key: %w", err) - } - if decoded, err := hex.DecodeString(oauthSigningKeyStr); err != nil || len(decoded) != len(options.OAuthSigningKey) { - b := make([]byte, len(options.OAuthSigningKey)) - _, err := rand.Read(b) - if err != nil { - return xerrors.Errorf("generate fresh oauth signing key: %w", err) - } - - oauthSigningKeyStr = hex.EncodeToString(b) - err = tx.UpsertOAuthSigningKey(ctx, oauthSigningKeyStr) - if err != nil { - return xerrors.Errorf("insert freshly generated oauth signing key to database: %w", err) - } - } - - oauthKeyBytes, err := hex.DecodeString(oauthSigningKeyStr) - if err != nil { - return xerrors.Errorf("decode oauth signing key from database: %w", err) - } - if len(oauthKeyBytes) != len(options.OAuthSigningKey) { - return xerrors.Errorf("oauth signing key in database is not the correct length, expect %d got %d", len(options.OAuthSigningKey), len(oauthKeyBytes)) - } - copy(options.OAuthSigningKey[:], oauthKeyBytes) - if options.OAuthSigningKey == [32]byte{} { - return xerrors.Errorf("oauth signing key in database is empty") - } - - // Read the coordinator resume token signing key from the - // database. - resumeTokenKey, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, tx) - if err != nil { - return xerrors.Errorf("get coordinator resume token key from database: %w", err) - } - options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeTokenKey, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry) - return nil }, nil) if err != nil { - return err + return xerrors.Errorf("set deployment id: %w", err) } + resumeKeycache, err := cryptokeys.NewSigningCache(logger, options.Database, database.CryptoKeyFeatureTailnetResume) + if err != nil { + return xerrors.Errorf("create resume token key cache: %w", err) + } + + options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeKeycache, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry) + options.RuntimeConfig = runtimeconfig.NewManager() // This should be output before the logs start streaming. diff --git a/coderd/coderd.go b/coderd/coderd.go index c8577219c7b15..dabeff0a8097d 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -186,9 +186,7 @@ type Options struct { TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore] UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore] AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore] - // AppSecurityKey is the crypto key used to sign and encrypt tokens related to // workspace applications. It consists of both a signing and encryption key. - AppSecurityKey workspaceapps.SecurityKey // CoordinatorResumeTokenProvider is used to provide and validate resume // tokens issued by and passed to the coordinator DRPC API. CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider @@ -445,6 +443,14 @@ func New(options *Options) *API { if err != nil { panic(xerrors.Errorf("get deployment ID: %w", err)) } + appSigningKeyCache, err := cryptokeys.NewSigningCache(options.Logger.Named("app_signing_key_cache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsToken) + if err != nil { + options.Logger.Fatal(ctx, "failed to initialize app signing key cache", slog.Error(err)) + } + appEncryptingKeyCache, err := cryptokeys.NewEncryptionCache(options.Logger.Named("app_encrypting_key_cache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsAPIKey) + if err != nil { + options.Logger.Fatal(ctx, "failed to initialize app encrypting key cache", slog.Error(err)) + } api := &API{ ctx: ctx, cancel: cancel, @@ -465,7 +471,7 @@ func New(options *Options) *API { options.DeploymentValues, oauthConfigs, options.AgentInactiveDisconnectTimeout, - options.AppSecurityKey, + appSigningKeyCache, ), metricsCache: metricsCache, Auditor: atomic.Pointer[audit.Auditor]{}, @@ -620,6 +626,7 @@ func New(options *Options) *API { if err != nil { api.Logger.Fatal(api.ctx, "failed to initialize oauth convert key cache", slog.Error(err)) } + api.workspaceAppsKeyCache = appEncryptingKeyCache api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ Database: options.Database, @@ -640,9 +647,6 @@ func New(options *Options) *API { options.WorkspaceAppsStatsCollectorOptions.Reporter = api.statsReporter } - if options.AppSecurityKey.IsZero() { - api.Logger.Fatal(api.ctx, "app security key cannot be zero") - } api.workspaceAppServer = &workspaceapps.Server{ Logger: workspaceAppsLogger, @@ -654,11 +658,12 @@ func New(options *Options) *API { SignedTokenProvider: api.WorkspaceAppsProvider, AgentProvider: api.agentProvider, - AppSecurityKey: options.AppSecurityKey, StatsCollector: workspaceapps.NewStatsCollector(options.WorkspaceAppsStatsCollectorOptions), - DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), - SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), + DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), + SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), + Signer: appSigningKeyCache, + EncryptingKeyManager: appEncryptingKeyCache, } apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{ @@ -1405,7 +1410,8 @@ type API struct { // resumeTokenKeycache is used to fetch and cache keys used for signing JWTs // oauthConvertKeycache is used to fetch and cache keys used for signing JWTs // during OAuth conversions. See userauth.go.convertUserToOauth. - oauthConvertKeycache cryptokeys.SigningKeycache + oauthConvertKeycache cryptokeys.SigningKeycache + workspaceAppsKeyCache cryptokeys.EncryptionKeycache } // Close waits for all WebSocket connections to drain before returning. diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 05c31f35bd20a..3147e7bf59e92 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -90,10 +90,6 @@ import ( "github.com/coder/coder/v2/testutil" ) -// AppSecurityKey is a 96-byte key used to sign JWTs and encrypt JWEs for -// workspace app tokens in tests. -var AppSecurityKey = must(workspaceapps.KeyFromString("6465616e207761732068657265206465616e207761732068657265206465616e207761732068657265206465616e207761732068657265206465616e207761732068657265206465616e207761732068657265206465616e2077617320686572")) - type Options struct { // AccessURL denotes a custom access URL. By default we use the httptest // server's URL. Setting this may result in unexpected behavior (especially @@ -525,7 +521,6 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can DeploymentOptions: codersdk.DeploymentOptionsWithoutSecrets(options.DeploymentValues.Options()), UpdateCheckOptions: options.UpdateCheckOptions, SwaggerEndpoint: options.SwaggerEndpoint, - AppSecurityKey: AppSecurityKey, SSHConfig: options.ConfigSSH, HealthcheckFunc: options.HealthcheckFunc, HealthcheckTimeout: options.HealthcheckTimeout, diff --git a/coderd/jwtutils/jwe.go b/coderd/jwtutils/jwe.go index d03816a477a26..c3952b7be25ad 100644 --- a/coderd/jwtutils/jwe.go +++ b/coderd/jwtutils/jwe.go @@ -15,6 +15,11 @@ const ( encryptContentAlgo = jose.A256GCM ) +type EncryptingKeyManager interface { + EncryptKeyProvider + DecryptKeyProvider +} + type EncryptKeyProvider interface { EncryptingKey(ctx context.Context) (id string, key interface{}, err error) } @@ -65,6 +70,12 @@ func Encrypt(ctx context.Context, e EncryptKeyProvider, claims Claims) (string, return compact, nil } +func WithDecryptExpected(expected jwt.Expected) func(*DecryptOptions) { + return func(opts *DecryptOptions) { + opts.RegisteredClaims = expected + } +} + // DecryptOptions are options for decrypting a JWE. type DecryptOptions struct { RegisteredClaims jwt.Expected @@ -119,3 +130,30 @@ func Decrypt(ctx context.Context, d DecryptKeyProvider, token string, claims Cla return claims.Validate(options.RegisteredClaims) } + +type StaticKeyManager struct { + ID string + Key interface{} +} + +func (s StaticKeyManager) SigningKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKeyManager) VerifyingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} + +func (s StaticKeyManager) EncryptingKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKeyManager) DecryptingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} diff --git a/coderd/jwtutils/jws.go b/coderd/jwtutils/jws.go index 73f35e672492d..6ea29e74a6956 100644 --- a/coderd/jwtutils/jws.go +++ b/coderd/jwtutils/jws.go @@ -24,6 +24,27 @@ const ( signingAlgo = jose.HS512 ) +type StaticKeyManager struct { + ID string + Key interface{} +} + +func (s StaticKeyManager) SigningKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKeyManager) VerifyingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} + +type SigningKeyManager interface { + SigningKeyProvider + VerifyKeyProvider +} + type SigningKeyProvider interface { SigningKey(ctx context.Context) (id string, key interface{}, err error) } @@ -75,6 +96,12 @@ type VerifyOptions struct { SignatureAlgorithm jose.SignatureAlgorithm } +func WithVerifyExpected(expected jwt.Expected) func(*VerifyOptions) { + return func(opts *VerifyOptions) { + opts.RegisteredClaims = expected + } +} + // Verify verifies that a token was signed by the provided key. It unmarshals into the provided claims. func Verify(ctx context.Context, v VerifyKeyProvider, token string, claims Claims, opts ...func(*VerifyOptions)) error { options := VerifyOptions{ diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 4b1af869cc007..8767d2412e753 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -853,7 +853,7 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R ) if resumeToken != "" { var err error - peerID, err = api.Options.CoordinatorResumeTokenProvider.VerifyResumeToken(resumeToken) + peerID, err = api.Options.CoordinatorResumeTokenProvider.VerifyResumeToken(ctx, resumeToken) if err != nil { httpapi.Write(ctx, rw, http.StatusUnauthorized, codersdk.Response{ Message: workspacesdk.CoordinateAPIInvalidResumeToken, diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 906333456ae70..9d7226e2e5919 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -36,6 +36,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/database/pubsub" "github.com/coder/coder/v2/coderd/externalauth" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/coder/v2/codersdk/workspacesdk" @@ -531,20 +532,20 @@ func newResumeTokenRecordingProvider(t testing.TB, underlying tailnet.ResumeToke } } -func (r *resumeTokenRecordingProvider) GenerateResumeToken(peerID uuid.UUID) (*tailnetproto.RefreshResumeTokenResponse, error) { +func (r *resumeTokenRecordingProvider) GenerateResumeToken(ctx context.Context, peerID uuid.UUID) (*tailnetproto.RefreshResumeTokenResponse, error) { select { case r.generateCalls <- peerID: - return r.ResumeTokenProvider.GenerateResumeToken(peerID) + return r.ResumeTokenProvider.GenerateResumeToken(ctx, peerID) default: r.t.Error("generateCalls full") return nil, xerrors.New("generateCalls full") } } -func (r *resumeTokenRecordingProvider) VerifyResumeToken(token string) (uuid.UUID, error) { +func (r *resumeTokenRecordingProvider) VerifyResumeToken(ctx context.Context, token string) (uuid.UUID, error) { select { case r.verifyCalls <- token: - return r.ResumeTokenProvider.VerifyResumeToken(token) + return r.ResumeTokenProvider.VerifyResumeToken(ctx, token) default: r.t.Error("verifyCalls full") return uuid.Nil, xerrors.New("verifyCalls full") @@ -557,10 +558,14 @@ func TestWorkspaceAgentClientCoordinate_ResumeToken(t *testing.T) { logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() + mgr := jwtutils.StaticKeyManager{ + ID: uuid.New().String(), + Key: resumeTokenSigningKey, + } require.NoError(t, err) resumeTokenProvider := newResumeTokenRecordingProvider( t, - tailnet.NewResumeTokenKeyProvider(resumeTokenSigningKey, clock, time.Hour), + tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour), ) client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ Coordinator: tailnet.NewCoordinator(logger), diff --git a/coderd/workspaceapps.go b/coderd/workspaceapps.go index d2fa11b9ea2ea..347edda3cece8 100644 --- a/coderd/workspaceapps.go +++ b/coderd/workspaceapps.go @@ -16,6 +16,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/coderd/workspaceapps/appurl" @@ -122,8 +123,7 @@ func (api *API) workspaceApplicationAuth(rw http.ResponseWriter, r *http.Request return } - // Encrypt the API key. - encryptedAPIKey, err := api.AppSecurityKey.EncryptAPIKey(workspaceapps.EncryptedAPIKeyPayload{ + encryptedAPIKey, err := jwtutils.Encrypt(ctx, api.workspaceAppsKeyCache, workspaceapps.EncryptedAPIKeyPayload{ APIKey: cookie.Value, }) if err != nil { diff --git a/coderd/workspaceapps/db.go b/coderd/workspaceapps/db.go index 1b369cf6d6ef4..e7c916372a7f8 100644 --- a/coderd/workspaceapps/db.go +++ b/coderd/workspaceapps/db.go @@ -18,9 +18,11 @@ import ( "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/codersdk" + "github.com/go-jose/go-jose/v4/jwt" ) // DBTokenProvider provides authentication and authorization for workspace apps @@ -35,12 +37,12 @@ type DBTokenProvider struct { DeploymentValues *codersdk.DeploymentValues OAuth2Configs *httpmw.OAuth2Configs WorkspaceAgentInactiveTimeout time.Duration - SigningKey SecurityKey + Signer jwtutils.SigningKeyManager } var _ SignedTokenProvider = &DBTokenProvider{} -func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authorizer, db database.Store, cfg *codersdk.DeploymentValues, oauth2Cfgs *httpmw.OAuth2Configs, workspaceAgentInactiveTimeout time.Duration, signingKey SecurityKey) SignedTokenProvider { +func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authorizer, db database.Store, cfg *codersdk.DeploymentValues, oauth2Cfgs *httpmw.OAuth2Configs, workspaceAgentInactiveTimeout time.Duration, signer jwtutils.SigningKeyManager) SignedTokenProvider { if workspaceAgentInactiveTimeout == 0 { workspaceAgentInactiveTimeout = 1 * time.Minute } @@ -53,12 +55,12 @@ func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authoriz DeploymentValues: cfg, OAuth2Configs: oauth2Cfgs, WorkspaceAgentInactiveTimeout: workspaceAgentInactiveTimeout, - SigningKey: signingKey, + Signer: signer, } } func (p *DBTokenProvider) FromRequest(r *http.Request) (*SignedToken, bool) { - return FromRequest(r, p.SigningKey) + return FromRequest(r, p.Signer) } func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *http.Request, issueReq IssueTokenRequest) (*SignedToken, string, bool) { @@ -70,7 +72,7 @@ func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r * dangerousSystemCtx := dbauthz.AsSystemRestricted(ctx) appReq := issueReq.AppRequest.Normalize() - err := appReq.Validate() + err := appReq.Check() if err != nil { WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "invalid app request") return nil, "", false @@ -211,8 +213,10 @@ func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r * } // Sign the token. - token.Expiry = time.Now().Add(DefaultTokenExpiry) - tokenStr, err := p.SigningKey.SignToken(token) + token.Claims = jwt.Claims{ + Expiry: jwt.NewNumericDate(time.Now().Add(DefaultTokenExpiry)), + } + tokenStr, err := jwtutils.Sign(ctx, p.Signer, token) if err != nil { WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "generate token") return nil, "", false diff --git a/coderd/workspaceapps/db_test.go b/coderd/workspaceapps/db_test.go index 6c5a0212aff2b..fb60b4dc07749 100644 --- a/coderd/workspaceapps/db_test.go +++ b/coderd/workspaceapps/db_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -276,15 +277,17 @@ func Test_ResolveRequest(t *testing.T) { _ = w.Body.Close() require.Equal(t, &workspaceapps.SignedToken{ + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(token.Expiry.Time()), + }, Request: req, - Expiry: token.Expiry, // ignored to avoid flakiness UserID: me.ID, WorkspaceID: workspace.ID, AgentID: agentID, AppURL: appURL, }, token) require.NotZero(t, token.Expiry) - require.WithinDuration(t, time.Now().Add(workspaceapps.DefaultTokenExpiry), token.Expiry, time.Minute) + require.WithinDuration(t, time.Now().Add(workspaceapps.DefaultTokenExpiry), token.Expiry.Time(), time.Minute) // Check that the token was set in the response and is valid. require.Len(t, w.Cookies(), 1) @@ -292,10 +295,10 @@ func Test_ResolveRequest(t *testing.T) { require.Equal(t, codersdk.SignedAppTokenCookie, cookie.Name) require.Equal(t, req.BasePath, cookie.Path) - parsedToken, err := api.AppSecurityKey.VerifySignedToken(cookie.Value) + parsedToken, err := api.workspaceAppsKeyCache.VerifySignedToken(cookie.Value) require.NoError(t, err) // normalize expiry - require.WithinDuration(t, token.Expiry, parsedToken.Expiry, 2*time.Second) + require.WithinDuration(t, token.Expiry.Time(), parsedToken.Expiry.Time(), 2*time.Second) parsedToken.Expiry = token.Expiry require.Equal(t, token, &parsedToken) @@ -314,7 +317,7 @@ func Test_ResolveRequest(t *testing.T) { }) require.True(t, ok) // normalize expiry - require.WithinDuration(t, token.Expiry, secondToken.Expiry, 2*time.Second) + require.WithinDuration(t, token.Expiry.Time(), secondToken.Expiry.Time(), 2*time.Second) secondToken.Expiry = token.Expiry require.Equal(t, token, secondToken) } @@ -540,7 +543,9 @@ func Test_ResolveRequest(t *testing.T) { // App name differs AppSlugOrPort: appNamePublic, }).Normalize(), - Expiry: time.Now().Add(time.Minute), + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(time.Now().Add(time.Minute)), + }, UserID: me.ID, WorkspaceID: workspace.ID, AgentID: agentID, diff --git a/coderd/workspaceapps/provider.go b/coderd/workspaceapps/provider.go index 8d4b7fd149800..1887036e35cbf 100644 --- a/coderd/workspaceapps/provider.go +++ b/coderd/workspaceapps/provider.go @@ -38,7 +38,7 @@ type ResolveRequestOptions struct { func ResolveRequest(rw http.ResponseWriter, r *http.Request, opts ResolveRequestOptions) (*SignedToken, bool) { appReq := opts.AppRequest.Normalize() - err := appReq.Validate() + err := appReq.Check() if err != nil { // This is a 500 since it's a coder server or proxy that's making this // request struct based on details from the request. The values should @@ -79,7 +79,7 @@ func ResolveRequest(rw http.ResponseWriter, r *http.Request, opts ResolveRequest Name: codersdk.SignedAppTokenCookie, Value: tokenStr, Path: appReq.BasePath, - Expires: token.Expiry, + Expires: token.Expiry.Time(), }) return token, true diff --git a/coderd/workspaceapps/proxy.go b/coderd/workspaceapps/proxy.go index 69f1aadca49b2..bf5e5bd517470 100644 --- a/coderd/workspaceapps/proxy.go +++ b/coderd/workspaceapps/proxy.go @@ -11,8 +11,10 @@ import ( "strconv" "strings" "sync" + "time" "github.com/go-chi/chi/v5" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" "go.opentelemetry.io/otel/trace" "nhooyr.io/websocket" @@ -22,6 +24,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/tracing" "github.com/coder/coder/v2/coderd/util/slice" "github.com/coder/coder/v2/coderd/workspaceapps/appurl" @@ -97,8 +100,9 @@ type Server struct { HostnameRegex *regexp.Regexp RealIPConfig *httpmw.RealIPConfig - SignedTokenProvider SignedTokenProvider - AppSecurityKey SecurityKey + SignedTokenProvider SignedTokenProvider + Signer jwtutils.SigningKeyManager + EncryptingKeyManager jwtutils.EncryptingKeyManager // DisablePathApps disables path-based apps. This is a security feature as path // based apps share the same cookie as the dashboard, and are susceptible to XSS @@ -176,7 +180,10 @@ func (s *Server) handleAPIKeySmuggling(rw http.ResponseWriter, r *http.Request, } // Exchange the encoded API key for a real one. - token, err := s.AppSecurityKey.DecryptAPIKey(encryptedAPIKey) + var payload EncryptedAPIKeyPayload + err := jwtutils.Decrypt(ctx, s.EncryptingKeyManager, encryptedAPIKey, &payload, jwtutils.WithDecryptExpected(jwt.Expected{ + Time: time.Now(), + })) if err != nil { s.Logger.Debug(ctx, "could not decrypt smuggled workspace app API key", slog.Error(err)) site.RenderStaticErrorPage(rw, r, site.ErrorPageData{ @@ -225,7 +232,7 @@ func (s *Server) handleAPIKeySmuggling(rw http.ResponseWriter, r *http.Request, // server using the wrong value. http.SetCookie(rw, &http.Cookie{ Name: AppConnectSessionTokenCookieName(accessMethod), - Value: token, + Value: payload.APIKey, Domain: domain, Path: "/", MaxAge: 0, diff --git a/coderd/workspaceapps/request.go b/coderd/workspaceapps/request.go index 4f6a6f3a64e65..ade95c782c5d9 100644 --- a/coderd/workspaceapps/request.go +++ b/coderd/workspaceapps/request.go @@ -126,7 +126,7 @@ func (r Request) Normalize() Request { // Validate ensures the request is correct and contains the necessary // parameters. -func (r Request) Validate() error { +func (r Request) Check() error { switch r.AccessMethod { case AccessMethodPath, AccessMethodSubdomain, AccessMethodTerminal: default: diff --git a/coderd/workspaceapps/request_test.go b/coderd/workspaceapps/request_test.go index b6e4bb7a2e65f..fbabc840745e9 100644 --- a/coderd/workspaceapps/request_test.go +++ b/coderd/workspaceapps/request_test.go @@ -279,7 +279,7 @@ func Test_RequestValidate(t *testing.T) { if !c.noNormalize { req = c.req.Normalize() } - err := req.Validate() + err := req.Check() if c.errContains == "" { require.NoError(t, err) } else { diff --git a/coderd/workspaceapps/token.go b/coderd/workspaceapps/token.go index 33428b0e25f13..24b7ea9ba580b 100644 --- a/coderd/workspaceapps/token.go +++ b/coderd/workspaceapps/token.go @@ -1,18 +1,15 @@ package workspaceapps import ( - "encoding/base64" - "encoding/hex" - "encoding/json" "net/http" "strings" "time" - "github.com/go-jose/go-jose/v3" + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" - "golang.org/x/xerrors" - "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" ) @@ -25,11 +22,10 @@ const ( // contains the details of the workspace app that the token is valid for to // avoid database queries. type SignedToken struct { + jwt.Claims // Request details. Request `json:"request"` - // Trusted resolved details. - Expiry time.Time `json:"expiry"` // set by GenerateToken if unset UserID uuid.UUID `json:"user_id"` WorkspaceID uuid.UUID `json:"workspace_id"` AgentID uuid.UUID `json:"agent_id"` @@ -57,191 +53,14 @@ func (t SignedToken) MatchesRequest(req Request) bool { t.AppSlugOrPort == req.AppSlugOrPort } -// SecurityKey is used for signing and encrypting app tokens and API keys. -// -// The first 64 bytes of the key are used for signing tokens with HMAC-SHA256, -// and the last 32 bytes are used for encrypting API keys with AES-256-GCM. -// We use a single key for both operations to avoid having to store and manage -// two keys. -type SecurityKey [96]byte - -func (k SecurityKey) IsZero() bool { - return k == SecurityKey{} -} - -func (k SecurityKey) String() string { - return hex.EncodeToString(k[:]) -} - -func (k SecurityKey) signingKey() []byte { - return k[:64] -} - -func (k SecurityKey) encryptionKey() []byte { - return k[64:] -} - -func KeyFromString(str string) (SecurityKey, error) { - var key SecurityKey - decoded, err := hex.DecodeString(str) - if err != nil { - return key, xerrors.Errorf("decode key: %w", err) - } - if len(decoded) != len(key) { - return key, xerrors.Errorf("expected key to be %d bytes, got %d", len(key), len(decoded)) - } - copy(key[:], decoded) - - return key, nil -} - -// SignToken generates a signed workspace app token with the given payload. If -// the payload doesn't have an expiry, it will be set to the current time plus -// the default expiry. -func (k SecurityKey) SignToken(payload SignedToken) (string, error) { - if payload.Expiry.IsZero() { - payload.Expiry = time.Now().Add(DefaultTokenExpiry) - } - payloadBytes, err := json.Marshal(payload) - if err != nil { - return "", xerrors.Errorf("marshal payload to JSON: %w", err) - } - - signer, err := jose.NewSigner(jose.SigningKey{ - Algorithm: tokenSigningAlgorithm, - Key: k.signingKey(), - }, nil) - if err != nil { - return "", xerrors.Errorf("create signer: %w", err) - } - - signedObject, err := signer.Sign(payloadBytes) - if err != nil { - return "", xerrors.Errorf("sign payload: %w", err) - } - - serialized, err := signedObject.CompactSerialize() - if err != nil { - return "", xerrors.Errorf("serialize JWS: %w", err) - } - - return serialized, nil -} - -// VerifySignedToken parses a signed workspace app token with the given key and -// returns the payload. If the token is invalid or expired, an error is -// returned. -func (k SecurityKey) VerifySignedToken(str string) (SignedToken, error) { - object, err := jose.ParseSigned(str) - if err != nil { - return SignedToken{}, xerrors.Errorf("parse JWS: %w", err) - } - if len(object.Signatures) != 1 { - return SignedToken{}, xerrors.New("expected 1 signature") - } - if object.Signatures[0].Header.Algorithm != string(tokenSigningAlgorithm) { - return SignedToken{}, xerrors.Errorf("expected token signing algorithm to be %q, got %q", tokenSigningAlgorithm, object.Signatures[0].Header.Algorithm) - } - - output, err := object.Verify(k.signingKey()) - if err != nil { - return SignedToken{}, xerrors.Errorf("verify JWS: %w", err) - } - - var tok SignedToken - err = json.Unmarshal(output, &tok) - if err != nil { - return SignedToken{}, xerrors.Errorf("unmarshal payload: %w", err) - } - if tok.Expiry.Before(time.Now()) { - return SignedToken{}, xerrors.New("signed app token expired") - } - - return tok, nil -} - type EncryptedAPIKeyPayload struct { - APIKey string `json:"api_key"` - ExpiresAt time.Time `json:"expires_at"` -} - -// EncryptAPIKey encrypts an API key for subdomain token smuggling. -func (k SecurityKey) EncryptAPIKey(payload EncryptedAPIKeyPayload) (string, error) { - if payload.APIKey == "" { - return "", xerrors.New("API key is empty") - } - if payload.ExpiresAt.IsZero() { - // Very short expiry as these keys are only used once as part of an - // automatic redirection flow. - payload.ExpiresAt = dbtime.Now().Add(time.Minute) - } - - payloadBytes, err := json.Marshal(payload) - if err != nil { - return "", xerrors.Errorf("marshal payload: %w", err) - } - - // JWEs seem to apply a nonce themselves. - encrypter, err := jose.NewEncrypter( - jose.A256GCM, - jose.Recipient{ - Algorithm: apiKeyEncryptionAlgorithm, - Key: k.encryptionKey(), - }, - &jose.EncrypterOptions{ - Compression: jose.DEFLATE, - }, - ) - if err != nil { - return "", xerrors.Errorf("initializer jose encrypter: %w", err) - } - encryptedObject, err := encrypter.Encrypt(payloadBytes) - if err != nil { - return "", xerrors.Errorf("encrypt jwe: %w", err) - } - - encrypted := encryptedObject.FullSerialize() - return base64.RawURLEncoding.EncodeToString([]byte(encrypted)), nil -} - -// DecryptAPIKey undoes EncryptAPIKey and is used in the subdomain app handler. -func (k SecurityKey) DecryptAPIKey(encryptedAPIKey string) (string, error) { - encrypted, err := base64.RawURLEncoding.DecodeString(encryptedAPIKey) - if err != nil { - return "", xerrors.Errorf("base64 decode encrypted API key: %w", err) - } - - object, err := jose.ParseEncrypted(string(encrypted)) - if err != nil { - return "", xerrors.Errorf("parse encrypted API key: %w", err) - } - if object.Header.Algorithm != string(apiKeyEncryptionAlgorithm) { - return "", xerrors.Errorf("expected API key encryption algorithm to be %q, got %q", apiKeyEncryptionAlgorithm, object.Header.Algorithm) - } - - // Decrypt using the hashed secret. - decrypted, err := object.Decrypt(k.encryptionKey()) - if err != nil { - return "", xerrors.Errorf("decrypt API key: %w", err) - } - - // Unmarshal the payload. - var payload EncryptedAPIKeyPayload - if err := json.Unmarshal(decrypted, &payload); err != nil { - return "", xerrors.Errorf("unmarshal decrypted payload: %w", err) - } - - // Validate expiry. - if payload.ExpiresAt.Before(dbtime.Now()) { - return "", xerrors.New("encrypted API key expired") - } - - return payload.APIKey, nil + jwt.Claims + APIKey string `json:"api_key"` } // FromRequest returns the signed token from the request, if it exists and is // valid. The caller must check that the token matches the request. -func FromRequest(r *http.Request, key SecurityKey) (*SignedToken, bool) { +func FromRequest(r *http.Request, mgr jwtutils.SigningKeyManager) (*SignedToken, bool) { // Get all signed app tokens from the request. This includes the query // parameter and all matching cookies sent with the request. If there are // somehow multiple signed app token cookies, we want to try all of them @@ -270,8 +89,12 @@ func FromRequest(r *http.Request, key SecurityKey) (*SignedToken, bool) { tokens = tokens[:4] } + ctx := r.Context() for _, tokenStr := range tokens { - token, err := key.VerifySignedToken(tokenStr) + var token SignedToken + err := jwtutils.Verify(ctx, mgr, tokenStr, &token, jwtutils.WithVerifyExpected(jwt.Expected{ + Time: time.Now(), + })) if err == nil { req := token.Request.Normalize() if hasQueryParam && req.AccessMethod != AccessMethodTerminal { @@ -280,7 +103,7 @@ func FromRequest(r *http.Request, key SecurityKey) (*SignedToken, bool) { return nil, false } - err := req.Validate() + err := req.Check() if err == nil { // The request has a valid signed app token, which is a valid // token signed by us. The caller must check that it matches diff --git a/coderd/workspaceapps/token_test.go b/coderd/workspaceapps/token_test.go index c656ae2ab77b8..cc04ea8cf4ed0 100644 --- a/coderd/workspaceapps/token_test.go +++ b/coderd/workspaceapps/token_test.go @@ -1,22 +1,21 @@ package workspaceapps_test import ( - "fmt" + "crypto/rand" "net/http" "net/http/httptest" "testing" "time" "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/testutil" + "github.com/go-jose/go-jose/v4/jwt" - "github.com/go-jose/go-jose/v3" "github.com/google/uuid" "github.com/stretchr/testify/require" - "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database/dbtime" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" - "github.com/coder/coder/v2/cryptorand" ) func Test_TokenMatchesRequest(t *testing.T) { @@ -289,7 +288,10 @@ func Test_GenerateToken(t *testing.T) { t.Run("SetExpiry", func(t *testing.T) { t.Parallel() - tokenStr, err := coderdtest.AppSecurityKey.SignToken(workspaceapps.SignedToken{ + ctx := testutil.Context(t, testutil.WaitShort) + signer := newSigner(t) + + tokenStr, err := jwtutils.Sign(ctx, signer, workspaceapps.SignedToken{ Request: workspaceapps.Request{ AccessMethod: workspaceapps.AccessMethodPath, BasePath: "/app", @@ -299,7 +301,6 @@ func Test_GenerateToken(t *testing.T) { AppSlugOrPort: "qux", }, - Expiry: time.Time{}, UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), @@ -307,10 +308,11 @@ func Test_GenerateToken(t *testing.T) { }) require.NoError(t, err) - token, err := coderdtest.AppSecurityKey.VerifySignedToken(tokenStr) + var token workspaceapps.SignedToken + err = jwtutils.Verify(ctx, signer, tokenStr, &token) require.NoError(t, err) - require.WithinDuration(t, time.Now().Add(time.Minute), token.Expiry, 15*time.Second) + require.WithinDuration(t, time.Now().Add(time.Minute), token.Expiry.Time(), 15*time.Second) }) future := time.Now().Add(time.Hour) @@ -322,6 +324,9 @@ func Test_GenerateToken(t *testing.T) { { name: "OK1", token: workspaceapps.SignedToken{ + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(future), + }, Request: workspaceapps.Request{ AccessMethod: workspaceapps.AccessMethodPath, BasePath: "/app", @@ -331,7 +336,6 @@ func Test_GenerateToken(t *testing.T) { AppSlugOrPort: "qux", }, - Expiry: future, UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), @@ -341,6 +345,9 @@ func Test_GenerateToken(t *testing.T) { { name: "OK2", token: workspaceapps.SignedToken{ + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(future), + }, Request: workspaceapps.Request{ AccessMethod: workspaceapps.AccessMethodSubdomain, BasePath: "/", @@ -350,7 +357,6 @@ func Test_GenerateToken(t *testing.T) { AppSlugOrPort: "xuq", }, - Expiry: future, UserID: uuid.MustParse("6fa684a3-11aa-49fd-8512-ab527bd9b900"), WorkspaceID: uuid.MustParse("b2d816cc-505c-441d-afdf-dae01781bc0b"), AgentID: uuid.MustParse("6c4396e1-af88-4a8a-91a3-13ea54fc29fb"), @@ -360,6 +366,10 @@ func Test_GenerateToken(t *testing.T) { { name: "Expired", token: workspaceapps.SignedToken{ + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(time.Now().Add(-time.Hour)), + }, + Request: workspaceapps.Request{ AccessMethod: workspaceapps.AccessMethodSubdomain, BasePath: "/", @@ -369,7 +379,6 @@ func Test_GenerateToken(t *testing.T) { AppSlugOrPort: "qux", }, - Expiry: time.Now().Add(-time.Hour), UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), @@ -379,26 +388,29 @@ func Test_GenerateToken(t *testing.T) { }, } + signer := newSigner(t) for _, c := range cases { c := c t.Run(c.name, func(t *testing.T) { t.Parallel() - str, err := coderdtest.AppSecurityKey.SignToken(c.token) + ctx := testutil.Context(t, testutil.WaitShort) + str, err := jwtutils.Sign(ctx, signer, c.token) require.NoError(t, err) // Tokens aren't deterministic as they have a random nonce, so we // can't compare them directly. - token, err := coderdtest.AppSecurityKey.VerifySignedToken(str) + var token workspaceapps.SignedToken + err = jwtutils.Verify(ctx, signer, str, &token) if c.parseErrContains != "" { require.Error(t, err) require.ErrorContains(t, err, c.parseErrContains) } else { require.NoError(t, err) // normalize the expiry - require.WithinDuration(t, c.token.Expiry, token.Expiry, 10*time.Second) + require.WithinDuration(t, c.token.Expiry.Time(), token.Expiry.Time(), 10*time.Second) c.token.Expiry = token.Expiry require.Equal(t, c.token, token) } @@ -419,7 +431,13 @@ func Test_FromRequest(t *testing.T) { Value: "invalid", }) + ctx := testutil.Context(t, testutil.WaitShort) + signer := newSigner(t) + token := workspaceapps.SignedToken{ + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(time.Now().Add(time.Hour)), + }, Request: workspaceapps.Request{ AccessMethod: workspaceapps.AccessMethodSubdomain, BasePath: "/", @@ -429,7 +447,6 @@ func Test_FromRequest(t *testing.T) { AgentNameOrID: "agent", AppSlugOrPort: "app", }, - Expiry: time.Now().Add(time.Hour), UserID: uuid.New(), WorkspaceID: uuid.New(), AgentID: uuid.New(), @@ -438,16 +455,15 @@ func Test_FromRequest(t *testing.T) { // Add an expired cookie expired := token - expired.Expiry = time.Now().Add(time.Hour * -1) - expiredStr, err := coderdtest.AppSecurityKey.SignToken(token) + expired.Claims.Expiry = jwt.NewNumericDate(time.Now().Add(time.Hour * -1)) + expiredStr, err := jwtutils.Sign(ctx, signer, expired) require.NoError(t, err) r.AddCookie(&http.Cookie{ Name: codersdk.SignedAppTokenCookie, Value: expiredStr, }) - // Add a valid token - validStr, err := coderdtest.AppSecurityKey.SignToken(token) + validStr, err := jwtutils.Sign(ctx, signer, token) require.NoError(t, err) r.AddCookie(&http.Cookie{ @@ -455,147 +471,27 @@ func Test_FromRequest(t *testing.T) { Value: validStr, }) - signed, ok := workspaceapps.FromRequest(r, coderdtest.AppSecurityKey) + signed, ok := workspaceapps.FromRequest(r, signer) require.True(t, ok, "expected a token to be found") // Confirm it is the correct token. require.Equal(t, signed.UserID, token.UserID) }) } -// The ParseToken fn is tested quite thoroughly in the GenerateToken test as -// well. -func Test_ParseToken(t *testing.T) { - t.Parallel() - - t.Run("InvalidJWS", func(t *testing.T) { - t.Parallel() - - token, err := coderdtest.AppSecurityKey.VerifySignedToken("invalid") - require.Error(t, err) - require.ErrorContains(t, err, "parse JWS") - require.Equal(t, workspaceapps.SignedToken{}, token) - }) - - t.Run("VerifySignature", func(t *testing.T) { - t.Parallel() - - // Create a valid token using a different key. - var otherKey workspaceapps.SecurityKey - copy(otherKey[:], coderdtest.AppSecurityKey[:]) - for i := range otherKey { - otherKey[i] ^= 0xff - } - require.NotEqual(t, coderdtest.AppSecurityKey, otherKey) - - tokenStr, err := otherKey.SignToken(workspaceapps.SignedToken{ - Request: workspaceapps.Request{ - AccessMethod: workspaceapps.AccessMethodPath, - BasePath: "/app", - UsernameOrID: "foo", - WorkspaceNameOrID: "bar", - AgentNameOrID: "baz", - AppSlugOrPort: "qux", - }, - - Expiry: time.Now().Add(time.Hour), - UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), - WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), - AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), - AppURL: "http://127.0.0.1:8080", - }) - require.NoError(t, err) - - // Verify the token is invalid. - token, err := coderdtest.AppSecurityKey.VerifySignedToken(tokenStr) - require.Error(t, err) - require.ErrorContains(t, err, "verify JWS") - require.Equal(t, workspaceapps.SignedToken{}, token) - }) - - t.Run("InvalidBody", func(t *testing.T) { - t.Parallel() - - // Create a signature for an invalid body. - signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS512, Key: coderdtest.AppSecurityKey[:64]}, nil) - require.NoError(t, err) - signedObject, err := signer.Sign([]byte("hi")) - require.NoError(t, err) - serialized, err := signedObject.CompactSerialize() - require.NoError(t, err) - - token, err := coderdtest.AppSecurityKey.VerifySignedToken(serialized) - require.Error(t, err) - require.ErrorContains(t, err, "unmarshal payload") - require.Equal(t, workspaceapps.SignedToken{}, token) - }) -} - -func TestAPIKeyEncryption(t *testing.T) { - t.Parallel() - - genAPIKey := func(t *testing.T) string { - id, _ := cryptorand.String(10) - secret, _ := cryptorand.String(22) +func newSigner(t *testing.T) jwtutils.SigningKeyManager { + t.Helper() - return fmt.Sprintf("%s-%s", id, secret) + return jwtutils.StaticKeyManager{ + ID: "test", + Key: generateSecret(t, 64), } +} - t.Run("OK", func(t *testing.T) { - t.Parallel() - - key := genAPIKey(t) - encrypted, err := coderdtest.AppSecurityKey.EncryptAPIKey(workspaceapps.EncryptedAPIKeyPayload{ - APIKey: key, - }) - require.NoError(t, err) - - decryptedKey, err := coderdtest.AppSecurityKey.DecryptAPIKey(encrypted) - require.NoError(t, err) - require.Equal(t, key, decryptedKey) - }) - - t.Run("Verifies", func(t *testing.T) { - t.Parallel() - - t.Run("Expiry", func(t *testing.T) { - t.Parallel() - - key := genAPIKey(t) - encrypted, err := coderdtest.AppSecurityKey.EncryptAPIKey(workspaceapps.EncryptedAPIKeyPayload{ - APIKey: key, - ExpiresAt: dbtime.Now().Add(-1 * time.Hour), - }) - require.NoError(t, err) - - decryptedKey, err := coderdtest.AppSecurityKey.DecryptAPIKey(encrypted) - require.Error(t, err) - require.ErrorContains(t, err, "expired") - require.Empty(t, decryptedKey) - }) - - t.Run("EncryptionKey", func(t *testing.T) { - t.Parallel() - - // Create a valid token using a different key. - var otherKey workspaceapps.SecurityKey - copy(otherKey[:], coderdtest.AppSecurityKey[:]) - for i := range otherKey { - otherKey[i] ^= 0xff - } - require.NotEqual(t, coderdtest.AppSecurityKey, otherKey) - - // Encrypt with the other key. - key := genAPIKey(t) - encrypted, err := otherKey.EncryptAPIKey(workspaceapps.EncryptedAPIKeyPayload{ - APIKey: key, - }) - require.NoError(t, err) +func generateSecret(t *testing.T, size int) []byte { + t.Helper() - // Decrypt with the original key. - decryptedKey, err := coderdtest.AppSecurityKey.DecryptAPIKey(encrypted) - require.Error(t, err) - require.ErrorContains(t, err, "decrypt API key") - require.Empty(t, decryptedKey) - }) - }) + secret := make([]byte, size) + _, err := rand.Read(secret) + require.NoError(t, err) + return secret } diff --git a/codersdk/workspacesdk/connector_internal_test.go b/codersdk/workspacesdk/connector_internal_test.go index 7a339a0079ba2..9599a56918056 100644 --- a/codersdk/workspacesdk/connector_internal_test.go +++ b/codersdk/workspacesdk/connector_internal_test.go @@ -25,6 +25,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/apiversion" "github.com/coder/coder/v2/coderd/httpapi" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/tailnet/proto" @@ -61,7 +62,7 @@ func TestTailnetAPIConnector_Disconnects(t *testing.T) { CoordPtr: &coordPtr, DERPMapUpdateFrequency: time.Millisecond, DERPMapFn: func() *tailcfg.DERPMap { return <-derpMapCh }, - NetworkTelemetryHandler: func(batch []*proto.TelemetryEvent) {}, + NetworkTelemetryHandler: func([]*proto.TelemetryEvent) {}, ResumeTokenProvider: tailnet.NewInsecureTestResumeTokenProvider(), }) require.NoError(t, err) @@ -165,13 +166,17 @@ func TestTailnetAPIConnector_ResumeToken(t *testing.T) { clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(resumeTokenSigningKey, clock, time.Hour) + mgr := jwtutils.StaticKeyManager{ + ID: uuid.New().String(), + Key: resumeTokenSigningKey, + } + resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour) svc, err := tailnet.NewClientService(tailnet.ClientServiceOptions{ Logger: logger, CoordPtr: &coordPtr, DERPMapUpdateFrequency: time.Millisecond, DERPMapFn: func() *tailcfg.DERPMap { return <-derpMapCh }, - NetworkTelemetryHandler: func(batch []*proto.TelemetryEvent) {}, + NetworkTelemetryHandler: func([]*proto.TelemetryEvent) {}, ResumeTokenProvider: resumeTokenProvider, }) require.NoError(t, err) @@ -190,7 +195,7 @@ func TestTailnetAPIConnector_ResumeToken(t *testing.T) { t.Logf("received resume token: %s", resumeToken) assert.Equal(t, expectResumeToken, resumeToken) if resumeToken != "" { - peerID, err = resumeTokenProvider.VerifyResumeToken(resumeToken) + peerID, err = resumeTokenProvider.VerifyResumeToken(ctx, resumeToken) assert.NoError(t, err, "failed to parse resume token") if err != nil { httpapi.Write(ctx, w, http.StatusUnauthorized, codersdk.Response{ @@ -280,13 +285,17 @@ func TestTailnetAPIConnector_ResumeTokenFailure(t *testing.T) { clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(resumeTokenSigningKey, clock, time.Hour) + mgr := jwtutils.StaticKeyManager{ + ID: uuid.New().String(), + Key: resumeTokenSigningKey, + } + resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour) svc, err := tailnet.NewClientService(tailnet.ClientServiceOptions{ Logger: logger, CoordPtr: &coordPtr, DERPMapUpdateFrequency: time.Millisecond, DERPMapFn: func() *tailcfg.DERPMap { return <-derpMapCh }, - NetworkTelemetryHandler: func(batch []*proto.TelemetryEvent) {}, + NetworkTelemetryHandler: func(_ []*proto.TelemetryEvent) {}, ResumeTokenProvider: resumeTokenProvider, }) require.NoError(t, err) diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index 2d07e2ef16848..f69046b33c78a 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -700,7 +700,6 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request) } httpapi.Write(ctx, rw, http.StatusCreated, wsproxysdk.RegisterWorkspaceProxyResponse{ - AppSecurityKey: api.AppSecurityKey.String(), DERPMeshKey: api.DERPServer.MeshKey(), DERPRegionID: regionID, DERPMap: api.AGPL.DERPMap(), diff --git a/enterprise/wsproxy/tokenprovider.go b/enterprise/wsproxy/tokenprovider.go index 38822a4e7a22d..45f955d3f2dec 100644 --- a/enterprise/wsproxy/tokenprovider.go +++ b/enterprise/wsproxy/tokenprovider.go @@ -7,6 +7,7 @@ import ( "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" ) @@ -18,18 +19,19 @@ type TokenProvider struct { AccessURL *url.URL AppHostname string - Client *wsproxysdk.Client - SecurityKey workspaceapps.SecurityKey - Logger slog.Logger + Client *wsproxysdk.Client + SigningKey jwtutils.SigningKeyManager + EncryptingKey jwtutils.EncryptingKeyManager + Logger slog.Logger } func (p *TokenProvider) FromRequest(r *http.Request) (*workspaceapps.SignedToken, bool) { - return workspaceapps.FromRequest(r, p.SecurityKey) + return workspaceapps.FromRequest(r, p.SigningKey) } func (p *TokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *http.Request, issueReq workspaceapps.IssueTokenRequest) (*workspaceapps.SignedToken, string, bool) { appReq := issueReq.AppRequest.Normalize() - err := appReq.Validate() + err := appReq.Check() if err != nil { workspaceapps.WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "invalid app request") return nil, "", false @@ -42,7 +44,8 @@ func (p *TokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *ht } // Check that it verifies properly and matches the string. - token, err := p.SecurityKey.VerifySignedToken(resp.SignedTokenStr) + var token workspaceapps.SignedToken + err = jwtutils.Verify(ctx, p.SigningKey, resp.SignedTokenStr, &token) if err != nil { workspaceapps.WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "failed to verify newly generated signed token") return nil, "", false diff --git a/tailnet/resume.go b/tailnet/resume.go index b9443064a37f9..799c049885b10 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -5,15 +5,16 @@ import ( "crypto/rand" "database/sql" "encoding/hex" - "encoding/json" "time" - "github.com/go-jose/go-jose/v3" + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" "golang.org/x/xerrors" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/tailnet/proto" "github.com/coder/quartz" ) @@ -37,12 +38,15 @@ func NewInsecureTestResumeTokenProvider() ResumeTokenProvider { if err != nil { panic(err) } - return NewResumeTokenKeyProvider(key, quartz.NewReal(), time.Hour) + return NewResumeTokenKeyProvider(jwtutils.StaticKeyManager{ + ID: uuid.New().String(), + Key: key, + }, quartz.NewReal(), time.Hour) } type ResumeTokenProvider interface { - GenerateResumeToken(peerID uuid.UUID) (*proto.RefreshResumeTokenResponse, error) - VerifyResumeToken(token string) (uuid.UUID, error) + GenerateResumeToken(ctx context.Context, peerID uuid.UUID) (*proto.RefreshResumeTokenResponse, error) + VerifyResumeToken(ctx context.Context, token string) (uuid.UUID, error) } type ResumeTokenSigningKey [64]byte @@ -98,12 +102,12 @@ func ResumeTokenSigningKeyFromDatabase(ctx context.Context, db ResumeTokenSignin } type ResumeTokenKeyProvider struct { - key ResumeTokenSigningKey + key jwtutils.SigningKeyManager clock quartz.Clock expiry time.Duration } -func NewResumeTokenKeyProvider(key ResumeTokenSigningKey, clock quartz.Clock, expiry time.Duration) ResumeTokenProvider { +func NewResumeTokenKeyProvider(key jwtutils.SigningKeyManager, clock quartz.Clock, expiry time.Duration) ResumeTokenProvider { if expiry <= 0 { expiry = DefaultResumeTokenExpiry } @@ -115,45 +119,27 @@ func NewResumeTokenKeyProvider(key ResumeTokenSigningKey, clock quartz.Clock, ex } type resumeTokenPayload struct { + jwt.Claims PeerID uuid.UUID `json:"sub"` Expiry int64 `json:"exp"` } -func (p ResumeTokenKeyProvider) GenerateResumeToken(peerID uuid.UUID) (*proto.RefreshResumeTokenResponse, error) { +func (p ResumeTokenKeyProvider) GenerateResumeToken(ctx context.Context, peerID uuid.UUID) (*proto.RefreshResumeTokenResponse, error) { exp := p.clock.Now().Add(p.expiry) payload := resumeTokenPayload{ PeerID: peerID, - Expiry: exp.Unix(), - } - payloadBytes, err := json.Marshal(payload) - if err != nil { - return nil, xerrors.Errorf("marshal payload to JSON: %w", err) - } - - signer, err := jose.NewSigner(jose.SigningKey{ - Algorithm: resumeTokenSigningAlgorithm, - Key: p.key[:], - }, &jose.SignerOptions{ - ExtraHeaders: map[jose.HeaderKey]interface{}{ - "kid": resumeTokenSigningKeyID.String(), + Claims: jwt.Claims{ + Expiry: jwt.NewNumericDate(exp), }, - }) - if err != nil { - return nil, xerrors.Errorf("create signer: %w", err) } - signedObject, err := signer.Sign(payloadBytes) + token, err := jwtutils.Sign(ctx, p.key, payload) if err != nil { return nil, xerrors.Errorf("sign payload: %w", err) } - serialized, err := signedObject.CompactSerialize() - if err != nil { - return nil, xerrors.Errorf("serialize JWS: %w", err) - } - return &proto.RefreshResumeTokenResponse{ - Token: serialized, + Token: token, RefreshIn: durationpb.New(p.expiry / 2), ExpiresAt: timestamppb.New(exp), }, nil @@ -162,35 +148,13 @@ func (p ResumeTokenKeyProvider) GenerateResumeToken(peerID uuid.UUID) (*proto.Re // VerifyResumeToken parses a signed tailnet resume token with the given key and // returns the payload. If the token is invalid or expired, an error is // returned. -func (p ResumeTokenKeyProvider) VerifyResumeToken(str string) (uuid.UUID, error) { - object, err := jose.ParseSigned(str) - if err != nil { - return uuid.Nil, xerrors.Errorf("parse JWS: %w", err) - } - if len(object.Signatures) != 1 { - return uuid.Nil, xerrors.New("expected 1 signature") - } - if object.Signatures[0].Header.Algorithm != string(resumeTokenSigningAlgorithm) { - return uuid.Nil, xerrors.Errorf("expected token signing algorithm to be %q, got %q", resumeTokenSigningAlgorithm, object.Signatures[0].Header.Algorithm) - } - if object.Signatures[0].Header.KeyID != resumeTokenSigningKeyID.String() { - return uuid.Nil, xerrors.Errorf("expected token key ID to be %q, got %q", resumeTokenSigningKeyID, object.Signatures[0].Header.KeyID) - } - - output, err := object.Verify(p.key[:]) - if err != nil { - return uuid.Nil, xerrors.Errorf("verify JWS: %w", err) - } - +func (p ResumeTokenKeyProvider) VerifyResumeToken(ctx context.Context, str string) (uuid.UUID, error) { var tok resumeTokenPayload - err = json.Unmarshal(output, &tok) + err := jwtutils.Verify(ctx, p.key, str, &tok, jwtutils.WithVerifyExpected(jwt.Expected{ + Time: p.clock.Now(), + })) if err != nil { - return uuid.Nil, xerrors.Errorf("unmarshal payload: %w", err) + return uuid.Nil, xerrors.Errorf("verify payload: %w", err) } - exp := time.Unix(tok.Expiry, 0) - if exp.Before(p.clock.Now()) { - return uuid.Nil, xerrors.New("signed resume token expired") - } - return tok.PeerID, nil } diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index 3f63887cbfef3..879b732689259 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -13,6 +13,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbmock" "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" @@ -121,17 +122,18 @@ func TestResumeTokenKeyProvider(t *testing.T) { t.Run("OK", func(t *testing.T) { t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) id := uuid.New() clock := quartz.NewMock(t) - provider := tailnet.NewResumeTokenKeyProvider(key, clock, tailnet.DefaultResumeTokenExpiry) - token, err := provider.GenerateResumeToken(id) + provider := tailnet.NewResumeTokenKeyProvider(newKeySigner(key), clock, tailnet.DefaultResumeTokenExpiry) + token, err := provider.GenerateResumeToken(ctx, id) require.NoError(t, err) require.NotNil(t, token) require.NotEmpty(t, token.Token) require.Equal(t, tailnet.DefaultResumeTokenExpiry/2, token.RefreshIn.AsDuration()) require.WithinDuration(t, clock.Now().Add(tailnet.DefaultResumeTokenExpiry), token.ExpiresAt.AsTime(), time.Second) - gotID, err := provider.VerifyResumeToken(token.Token) + gotID, err := provider.VerifyResumeToken(ctx, token.Token) require.NoError(t, err) require.Equal(t, id, gotID) }) @@ -139,10 +141,11 @@ func TestResumeTokenKeyProvider(t *testing.T) { t.Run("Expired", func(t *testing.T) { t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) id := uuid.New() clock := quartz.NewMock(t) - provider := tailnet.NewResumeTokenKeyProvider(key, clock, tailnet.DefaultResumeTokenExpiry) - token, err := provider.GenerateResumeToken(id) + provider := tailnet.NewResumeTokenKeyProvider(newKeySigner(key), clock, tailnet.DefaultResumeTokenExpiry) + token, err := provider.GenerateResumeToken(ctx, id) require.NoError(t, err) require.NotNil(t, token) require.NotEmpty(t, token.Token) @@ -152,30 +155,39 @@ func TestResumeTokenKeyProvider(t *testing.T) { // Advance time past expiry _ = clock.Advance(tailnet.DefaultResumeTokenExpiry + time.Second) - _, err = provider.VerifyResumeToken(token.Token) + _, err = provider.VerifyResumeToken(ctx, token.Token) require.ErrorContains(t, err, "expired") }) t.Run("InvalidToken", func(t *testing.T) { t.Parallel() - provider := tailnet.NewResumeTokenKeyProvider(key, quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) - _, err := provider.VerifyResumeToken("invalid") + ctx := testutil.Context(t, testutil.WaitShort) + provider := tailnet.NewResumeTokenKeyProvider(newKeySigner(key), quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) + _, err := provider.VerifyResumeToken(ctx, "invalid") require.ErrorContains(t, err, "parse JWS") }) t.Run("VerifyError", func(t *testing.T) { t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) // Generate a resume token with a different key otherKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - otherProvider := tailnet.NewResumeTokenKeyProvider(otherKey, quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) - token, err := otherProvider.GenerateResumeToken(uuid.New()) + otherProvider := tailnet.NewResumeTokenKeyProvider(newKeySigner(otherKey), quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) + token, err := otherProvider.GenerateResumeToken(ctx, uuid.New()) require.NoError(t, err) - provider := tailnet.NewResumeTokenKeyProvider(key, quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) - _, err = provider.VerifyResumeToken(token.Token) + provider := tailnet.NewResumeTokenKeyProvider(newKeySigner(key), quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) + _, err = provider.VerifyResumeToken(ctx, token.Token) require.ErrorContains(t, err, "verify JWS") }) } + +func newKeySigner(key tailnet.ResumeTokenSigningKey) jwtutils.SigningKeyManager { + return jwtutils.StaticKeyManager{ + ID: uuid.New().String(), + Key: key, + } +} diff --git a/tailnet/service.go b/tailnet/service.go index 28a054dd8d671..7f38f63a589b3 100644 --- a/tailnet/service.go +++ b/tailnet/service.go @@ -177,7 +177,7 @@ func (s *DRPCService) RefreshResumeToken(ctx context.Context, _ *proto.RefreshRe return nil, xerrors.New("no Stream ID") } - res, err := s.ResumeTokenProvider.GenerateResumeToken(streamID.ID) + res, err := s.ResumeTokenProvider.GenerateResumeToken(ctx, streamID.ID) if err != nil { return nil, xerrors.Errorf("generate resume token: %w", err) } From 7fe88ea6642ce4bb23720ca3971258440a0691ed Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 14 Oct 2024 19:50:59 +0000 Subject: [PATCH 05/39] fix migrations --- .../000262_site_to_crypto_key.up.sql | 52 ------------------- ...sql => 000264_cryptokey_features.down.sql} | 11 ++-- .../000264_cryptokey_features.up.sql | 18 +++++++ .../fixtures/000262_site_to_crypto_key.up.sql | 15 ------ .../fixtures/000264_cryptokey_features.up.sql | 40 ++++++++++++++ 5 files changed, 63 insertions(+), 73 deletions(-) delete mode 100644 coderd/database/migrations/000262_site_to_crypto_key.up.sql rename coderd/database/migrations/{000262_site_to_crypto_key.down.sql => 000264_cryptokey_features.down.sql} (51%) create mode 100644 coderd/database/migrations/000264_cryptokey_features.up.sql delete mode 100644 coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql create mode 100644 coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql diff --git a/coderd/database/migrations/000262_site_to_crypto_key.up.sql b/coderd/database/migrations/000262_site_to_crypto_key.up.sql deleted file mode 100644 index 3ca41ce95f76d..0000000000000 --- a/coderd/database/migrations/000262_site_to_crypto_key.up.sql +++ /dev/null @@ -1,52 +0,0 @@ --- Create a new enum type with the desired values -CREATE TYPE new_crypto_key_feature AS ENUM ( - 'workspace_apps_token', - 'workspace_apps_api_key', - 'oidc_convert', - 'tailnet_resume' -); - --- Drop the old type and rename the new one -ALTER TABLE crypto_keys - ALTER COLUMN feature TYPE new_crypto_key_feature - USING (feature::text::new_crypto_key_feature); - -DROP TYPE crypto_key_feature; - -ALTER TYPE new_crypto_key_feature RENAME TO crypto_key_feature; - --- Extract and decode the app_signing_key, then insert the first 64 bytes for workspace_apps_token -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -SELECT - 'workspace_apps_token'::crypto_key_feature, - 1, - encode(substring(decode(value, 'hex') from 1 for 64), 'base64'), - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - NULL -FROM site_configs -WHERE key = 'app_signing_key'; - --- Extract and decode the app_signing_key, then insert the last 32 bytes for workspace_apps_api_key -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -SELECT - 'workspace_apps_api_key'::crypto_key_feature, - 1, - encode(substring(decode(value, 'hex') from -32), 'base64'), - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - NULL -FROM site_configs -WHERE key = 'app_signing_key'; - --- Extract and decode the coordinator_resume_token_signing_key, then insert it for tailnet_resume feature -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -SELECT - 'tailnet_resume'::crypto_key_feature, - 1, - encode(decode(value, 'hex'), 'base64'), - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - NULL -FROM site_configs -WHERE key = 'coordinator_resume_token_signing_key'; diff --git a/coderd/database/migrations/000262_site_to_crypto_key.down.sql b/coderd/database/migrations/000264_cryptokey_features.down.sql similarity index 51% rename from coderd/database/migrations/000262_site_to_crypto_key.down.sql rename to coderd/database/migrations/000264_cryptokey_features.down.sql index 062b085cc2ea5..7cdd00d222da8 100644 --- a/coderd/database/migrations/000262_site_to_crypto_key.down.sql +++ b/coderd/database/migrations/000264_cryptokey_features.down.sql @@ -1,19 +1,18 @@ -- Step 1: Remove the new entries from crypto_keys table DELETE FROM crypto_keys -WHERE feature IN ('workspace_apps_token', 'workspace_apps_api_key', 'tailnet_resume') - AND sequence = 1; +WHERE feature IN ('workspace_apps_token', 'workspace_apps_api_key'); -CREATE TYPE crypto_key_feature_old AS ENUM ( +CREATE TYPE old_crypto_key_feature AS ENUM ( 'workspace_apps', 'oidc_convert', 'tailnet_resume' ); ALTER TABLE crypto_keys - ALTER COLUMN feature TYPE crypto_key_feature_old - USING (feature::text::crypto_key_feature_old); + ALTER COLUMN feature TYPE old_crypto_key_feature + USING (feature::text::old_crypto_key_feature); DROP TYPE crypto_key_feature; -ALTER TYPE crypto_key_feature_old RENAME TO crypto_key_feature; +ALTER TYPE old_crypto_key_feature RENAME TO crypto_key_feature; diff --git a/coderd/database/migrations/000264_cryptokey_features.up.sql b/coderd/database/migrations/000264_cryptokey_features.up.sql new file mode 100644 index 0000000000000..bca75d220d0c7 --- /dev/null +++ b/coderd/database/migrations/000264_cryptokey_features.up.sql @@ -0,0 +1,18 @@ +-- Create a new enum type with the desired values +CREATE TYPE new_crypto_key_feature AS ENUM ( + 'workspace_apps_token', + 'workspace_apps_api_key', + 'oidc_convert', + 'tailnet_resume' +); + +DELETE FROM crypto_keys WHERE feature = 'workspace_apps'; + +-- Drop the old type and rename the new one +ALTER TABLE crypto_keys + ALTER COLUMN feature TYPE new_crypto_key_feature + USING (feature::text::new_crypto_key_feature); + +DROP TYPE crypto_key_feature; + +ALTER TYPE new_crypto_key_feature RENAME TO crypto_key_feature; diff --git a/coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql b/coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql deleted file mode 100644 index f4e93636698da..0000000000000 --- a/coderd/database/migrations/testdata/fixtures/000262_site_to_crypto_key.up.sql +++ /dev/null @@ -1,15 +0,0 @@ -INSERT INTO site_configs (key, value) -VALUES ( - 'app_signing_key', - encode(gen_random_bytes(96), 'hex') -) -ON CONFLICT (key) DO UPDATE -SET value = EXCLUDED.value; - -INSERT INTO site_configs (key, value) -VALUES ( - 'coordinator_resume_token_signing_key', - encode(gen_random_bytes(32), 'hex') -) -ON CONFLICT (key) DO UPDATE -SET value = EXCLUDED.value; diff --git a/coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql new file mode 100644 index 0000000000000..5cb2cd4c95509 --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql @@ -0,0 +1,40 @@ +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'workspace_apps_token', + 1, + 'abc', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'workspace_apps_api_key', + 1, + 'def', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'oidc_convert', + 2, + 'ghi', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'tailnet_resume', + 2, + 'jkl', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + From 0323f791865e40c6c4dc3e27ed84afcda9d6647e Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 14 Oct 2024 22:10:16 +0000 Subject: [PATCH 06/39] Refactor key cache management for better clarity - Consolidates key cache handling by replacing legacy key cache references with central key caches. - Enhances modularity and maintainability by using consistent key management methods. - Removes redundant `StaticKeyManager` implementation for streamlined code. - Adjusts cryptographic key generation and cache utilization across critical components. --- coderd/coderd.go | 17 ++++-------- coderd/coderdtest/coderdtest.go | 14 ++++++++-- coderd/cryptokeys/rotate.go | 4 +++ coderd/jwtutils/jwe.go | 27 ------------------- coderd/userauth.go | 4 +-- coderd/workspaceagents_test.go | 2 +- coderd/workspaceapps.go | 2 +- coderd/workspaceapps/db.go | 3 ++- coderd/workspaceapps/db_test.go | 10 ++++--- coderd/workspaceapps/token_test.go | 3 ++- .../workspacesdk/connector_internal_test.go | 4 +-- enterprise/wsproxy/wsproxy.go | 24 ++++++++--------- tailnet/resume_test.go | 2 +- 13 files changed, 50 insertions(+), 66 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index dabeff0a8097d..70078bac5c53c 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -250,6 +250,11 @@ type Options struct { // OneTimePasscodeValidityPeriod specifies how long a one time passcode should be valid for. OneTimePasscodeValidityPeriod time.Duration + + // Keycaches + AppSigningKeyCache cryptokeys.SigningKeycache + AppEncryptionKeyCache cryptokeys.EncryptionKeycache + OIDCConvertKeyCache cryptokeys.SigningKeycache } // @title Coder API @@ -622,12 +627,6 @@ func New(options *Options) *API { api.Logger.Fatal(api.ctx, "start key rotator", slog.Error(err)) } - api.oauthConvertKeycache, err = cryptokeys.NewSigningCache(api.Logger.Named("oauth_convert_keycache"), api.Database, database.CryptoKeyFeatureOIDCConvert) - if err != nil { - api.Logger.Fatal(api.ctx, "failed to initialize oauth convert key cache", slog.Error(err)) - } - api.workspaceAppsKeyCache = appEncryptingKeyCache - api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ Database: options.Database, Logger: options.Logger.Named("workspacestats"), @@ -1406,12 +1405,6 @@ type API struct { // dbRolluper rolls up template usage stats from raw agent and app // stats. This is used to provide insights in the WebUI. dbRolluper *dbrollup.Rolluper - - // resumeTokenKeycache is used to fetch and cache keys used for signing JWTs - // oauthConvertKeycache is used to fetch and cache keys used for signing JWTs - // during OAuth conversions. See userauth.go.convertUserToOauth. - oauthConvertKeycache cryptokeys.SigningKeycache - workspaceAppsKeyCache cryptokeys.EncryptionKeycache } // Close waits for all WebSocket connections to drain before returning. diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 3147e7bf59e92..1ea15ea3ddce4 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -55,6 +55,7 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/autobuild" "github.com/coder/coder/v2/coderd/awsidentity" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbauthz" @@ -157,8 +158,7 @@ type Options struct { DatabaseRolluper *dbrollup.Rolluper WorkspaceUsageTrackerFlush chan int WorkspaceUsageTrackerTick chan time.Time - - NotificationsEnqueuer notifications.Enqueuer + NotificationsEnqueuer notifications.Enqueuer } // New constructs a codersdk client connected to an in-memory API instance. @@ -326,6 +326,13 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can } auditor.Store(&options.Auditor) + oidcConvertKeyCache, err := cryptokeys.NewSigningCache(options.Logger.Named("oidc_convert_keycache"), options.Database, database.CryptoKeyFeatureOIDCConvert) + require.NoError(t, err) + appSigningKeyCache, err := cryptokeys.NewSigningCache(options.Logger.Named("app_signing_keycache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsToken) + require.NoError(t, err) + appEncryptionKeyCache, err := cryptokeys.NewEncryptionCache(options.Logger.Named("app_encryption_keycache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsAPIKey) + require.NoError(t, err) + ctx, cancelFunc := context.WithCancel(context.Background()) lifecycleExecutor := autobuild.NewExecutor( ctx, @@ -533,6 +540,9 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can WorkspaceUsageTracker: wuTracker, NotificationsEnqueuer: options.NotificationsEnqueuer, OneTimePasscodeValidityPeriod: options.OneTimePasscodeValidityPeriod, + AppSigningKeyCache: appSigningKeyCache, + AppEncryptionKeyCache: appEncryptionKeyCache, + OIDCConvertKeyCache: oidcConvertKeyCache, } } diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index f5e36eca1da6c..e9902693f0191 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -231,6 +231,8 @@ func generateNewSecret(feature database.CryptoKeyFeature) (string, error) { switch feature { case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return generateKey(32) + case database.CryptoKeyFeatureWorkspaceAppsToken: + return generateKey(64) case database.CryptoKeyFeatureOIDCConvert: return generateKey(64) case database.CryptoKeyFeatureTailnetResume: @@ -252,6 +254,8 @@ func tokenDuration(feature database.CryptoKeyFeature) time.Duration { switch feature { case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return WorkspaceAppsTokenDuration + case database.CryptoKeyFeatureWorkspaceAppsToken: + return WorkspaceAppsTokenDuration case database.CryptoKeyFeatureOIDCConvert: return OIDCConvertTokenDuration case database.CryptoKeyFeatureTailnetResume: diff --git a/coderd/jwtutils/jwe.go b/coderd/jwtutils/jwe.go index c3952b7be25ad..5ac62946f52b7 100644 --- a/coderd/jwtutils/jwe.go +++ b/coderd/jwtutils/jwe.go @@ -130,30 +130,3 @@ func Decrypt(ctx context.Context, d DecryptKeyProvider, token string, claims Cla return claims.Validate(options.RegisteredClaims) } - -type StaticKeyManager struct { - ID string - Key interface{} -} - -func (s StaticKeyManager) SigningKey(_ context.Context) (string, interface{}, error) { - return s.ID, s.Key, nil -} - -func (s StaticKeyManager) VerifyingKey(_ context.Context, id string) (interface{}, error) { - if id != s.ID { - return nil, xerrors.Errorf("invalid id %q", id) - } - return s.Key, nil -} - -func (s StaticKeyManager) EncryptingKey(_ context.Context) (string, interface{}, error) { - return s.ID, s.Key, nil -} - -func (s StaticKeyManager) DecryptingKey(_ context.Context, id string) (interface{}, error) { - if id != s.ID { - return nil, xerrors.Errorf("invalid id %q", id) - } - return s.Key, nil -} diff --git a/coderd/userauth.go b/coderd/userauth.go index 25e9b5bd88a0e..a702ffb75f79b 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -167,7 +167,7 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { ToLoginType: req.ToType, } - token, err := jwtutils.Sign(dbauthz.AsKeyRotator(ctx), api.oauthConvertKeycache, claims) + token, err := jwtutils.Sign(dbauthz.AsKeyRotator(ctx), api.OIDCConvertKeyCache, claims) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error signing state jwt.", @@ -1676,7 +1676,7 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data } } var claims OAuthConvertStateClaims - err = jwtutils.Verify(dbauthz.AsKeyRotator(ctx), api.oauthConvertKeycache, jwtCookie.Value, &claims) + err = jwtutils.Verify(dbauthz.AsKeyRotator(ctx), api.OIDCConvertKeyCache, jwtCookie.Value, &claims) if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) { // These errors are probably because the user is mixing 2 coder deployments. return database.User{}, idpsync.HTTPError{ diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 9d7226e2e5919..225ec608f783a 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -560,7 +560,7 @@ func TestWorkspaceAgentClientCoordinate_ResumeToken(t *testing.T) { resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() mgr := jwtutils.StaticKeyManager{ ID: uuid.New().String(), - Key: resumeTokenSigningKey, + Key: resumeTokenSigningKey[:], } require.NoError(t, err) resumeTokenProvider := newResumeTokenRecordingProvider( diff --git a/coderd/workspaceapps.go b/coderd/workspaceapps.go index 347edda3cece8..52bbcd3b0b70d 100644 --- a/coderd/workspaceapps.go +++ b/coderd/workspaceapps.go @@ -123,7 +123,7 @@ func (api *API) workspaceApplicationAuth(rw http.ResponseWriter, r *http.Request return } - encryptedAPIKey, err := jwtutils.Encrypt(ctx, api.workspaceAppsKeyCache, workspaceapps.EncryptedAPIKeyPayload{ + encryptedAPIKey, err := jwtutils.Encrypt(ctx, api.AppEncryptionKeyCache, workspaceapps.EncryptedAPIKeyPayload{ APIKey: cookie.Value, }) if err != nil { diff --git a/coderd/workspaceapps/db.go b/coderd/workspaceapps/db.go index e7c916372a7f8..03667e279d42e 100644 --- a/coderd/workspaceapps/db.go +++ b/coderd/workspaceapps/db.go @@ -13,6 +13,8 @@ import ( "golang.org/x/exp/slices" "golang.org/x/xerrors" + "github.com/go-jose/go-jose/v4/jwt" + "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" @@ -22,7 +24,6 @@ import ( "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/codersdk" - "github.com/go-jose/go-jose/v4/jwt" ) // DBTokenProvider provides authentication and authorization for workspace apps diff --git a/coderd/workspaceapps/db_test.go b/coderd/workspaceapps/db_test.go index fb60b4dc07749..e648cc38cddff 100644 --- a/coderd/workspaceapps/db_test.go +++ b/coderd/workspaceapps/db_test.go @@ -21,6 +21,7 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/coderd/workspaceapps/appurl" "github.com/coder/coder/v2/codersdk" @@ -295,7 +296,8 @@ func Test_ResolveRequest(t *testing.T) { require.Equal(t, codersdk.SignedAppTokenCookie, cookie.Name) require.Equal(t, req.BasePath, cookie.Path) - parsedToken, err := api.workspaceAppsKeyCache.VerifySignedToken(cookie.Value) + var parsedToken workspaceapps.SignedToken + err := jwtutils.Verify(ctx, api.AppSigningKeyCache, cookie.Value, &parsedToken) require.NoError(t, err) // normalize expiry require.WithinDuration(t, token.Expiry.Time(), parsedToken.Expiry.Time(), 2*time.Second) @@ -551,7 +553,8 @@ func Test_ResolveRequest(t *testing.T) { AgentID: agentID, AppURL: appURL, } - badTokenStr, err := api.AppSecurityKey.SignToken(badToken) + + badTokenStr, err := jwtutils.Sign(ctx, api.AppSigningKeyCache, badToken) require.NoError(t, err) req := (workspaceapps.Request{ @@ -594,7 +597,8 @@ func Test_ResolveRequest(t *testing.T) { require.Len(t, cookies, 1) require.Equal(t, cookies[0].Name, codersdk.SignedAppTokenCookie) require.NotEqual(t, cookies[0].Value, badTokenStr) - parsedToken, err := api.AppSecurityKey.VerifySignedToken(cookies[0].Value) + var parsedToken workspaceapps.SignedToken + err = jwtutils.Verify(ctx, api.AppSigningKeyCache, cookies[0].Value, &parsedToken) require.NoError(t, err) require.Equal(t, appNameOwner, parsedToken.AppSlugOrPort) }) diff --git a/coderd/workspaceapps/token_test.go b/coderd/workspaceapps/token_test.go index cc04ea8cf4ed0..aa51366d661dd 100644 --- a/coderd/workspaceapps/token_test.go +++ b/coderd/workspaceapps/token_test.go @@ -7,9 +7,10 @@ import ( "testing" "time" + "github.com/go-jose/go-jose/v4/jwt" + "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" - "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" "github.com/stretchr/testify/require" diff --git a/codersdk/workspacesdk/connector_internal_test.go b/codersdk/workspacesdk/connector_internal_test.go index 9599a56918056..db48f5ef7fc1b 100644 --- a/codersdk/workspacesdk/connector_internal_test.go +++ b/codersdk/workspacesdk/connector_internal_test.go @@ -168,7 +168,7 @@ func TestTailnetAPIConnector_ResumeToken(t *testing.T) { require.NoError(t, err) mgr := jwtutils.StaticKeyManager{ ID: uuid.New().String(), - Key: resumeTokenSigningKey, + Key: resumeTokenSigningKey[:], } resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour) svc, err := tailnet.NewClientService(tailnet.ClientServiceOptions{ @@ -287,7 +287,7 @@ func TestTailnetAPIConnector_ResumeTokenFailure(t *testing.T) { require.NoError(t, err) mgr := jwtutils.StaticKeyManager{ ID: uuid.New().String(), - Key: resumeTokenSigningKey, + Key: resumeTokenSigningKey[:], } resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour) svc, err := tailnet.NewClientService(tailnet.ClientServiceOptions{ diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index cb66c411d3c70..676ddfc25ef3b 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -31,6 +31,7 @@ import ( "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/cli/cliutil" "github.com/coder/coder/v2/coderd" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/tracing" @@ -92,7 +93,9 @@ type Options struct { // from the dashboardURL. This should only be used in development. AllowAllCors bool - StatsCollectorOptions workspaceapps.StatsCollectorOptions + StatsCollectorOptions workspaceapps.StatsCollectorOptions + WorkspaceAppsEncryptionKeycache cryptokeys.EncryptionKeycache + WorkspaceAppsSigningKeycache cryptokeys.SigningKeycache } func (o *Options) Validate() error { @@ -240,11 +243,6 @@ func New(ctx context.Context, opts *Options) (*Server, error) { return nil, xerrors.Errorf("handle register: %w", err) } - secKey, err := workspaceapps.KeyFromString(regResp.AppSecurityKey) - if err != nil { - return nil, xerrors.Errorf("parse app security key: %w", err) - } - agentProvider, err := coderd.NewServerTailnet(ctx, s.Logger, nil, @@ -277,14 +275,14 @@ func New(ctx context.Context, opts *Options) (*Server, error) { HostnameRegex: opts.AppHostnameRegex, RealIPConfig: opts.RealIPConfig, SignedTokenProvider: &TokenProvider{ - DashboardURL: opts.DashboardURL, - AccessURL: opts.AccessURL, - AppHostname: opts.AppHostname, - Client: client, - SecurityKey: secKey, - Logger: s.Logger.Named("proxy_token_provider"), + DashboardURL: opts.DashboardURL, + AccessURL: opts.AccessURL, + AppHostname: opts.AppHostname, + Client: client, + SigningKey: opts.WorkspaceAppsSigningKeycache, + EncryptingKey: opts.WorkspaceAppsEncryptionKeycache, + Logger: s.Logger.Named("proxy_token_provider"), }, - AppSecurityKey: secKey, DisablePathApps: opts.DisablePathApps, SecureAuthCookie: opts.SecureAuthCookie, diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index 879b732689259..da67f9a0aa6d2 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -188,6 +188,6 @@ func TestResumeTokenKeyProvider(t *testing.T) { func newKeySigner(key tailnet.ResumeTokenSigningKey) jwtutils.SigningKeyManager { return jwtutils.StaticKeyManager{ ID: uuid.New().String(), - Key: key, + Key: key[:], } } From 7413907a8b446eedcc4352de6d8189bfceb47489 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 14 Oct 2024 23:13:01 +0000 Subject: [PATCH 07/39] hm --- coderd/coderd.go | 14 +++----------- coderd/cryptokeys/rotate.go | 2 +- coderd/database/dbauthz/dbauthz.go | 29 ++++++++++++++++++++++++++--- coderd/workspaceapps/db.go | 8 ++++---- tailnet/resume.go | 9 +-------- 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 70078bac5c53c..1aa12443cf70f 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -448,14 +448,6 @@ func New(options *Options) *API { if err != nil { panic(xerrors.Errorf("get deployment ID: %w", err)) } - appSigningKeyCache, err := cryptokeys.NewSigningCache(options.Logger.Named("app_signing_key_cache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsToken) - if err != nil { - options.Logger.Fatal(ctx, "failed to initialize app signing key cache", slog.Error(err)) - } - appEncryptingKeyCache, err := cryptokeys.NewEncryptionCache(options.Logger.Named("app_encrypting_key_cache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsAPIKey) - if err != nil { - options.Logger.Fatal(ctx, "failed to initialize app encrypting key cache", slog.Error(err)) - } api := &API{ ctx: ctx, cancel: cancel, @@ -476,7 +468,7 @@ func New(options *Options) *API { options.DeploymentValues, oauthConfigs, options.AgentInactiveDisconnectTimeout, - appSigningKeyCache, + options.AppSigningKeyCache, ), metricsCache: metricsCache, Auditor: atomic.Pointer[audit.Auditor]{}, @@ -661,8 +653,8 @@ func New(options *Options) *API { DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), - Signer: appSigningKeyCache, - EncryptingKeyManager: appEncryptingKeyCache, + Signer: options.AppSigningKeyCache, + EncryptingKeyManager: options.AppEncryptionKeyCache, } apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{ diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index e9902693f0191..186285c4a21c7 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -56,7 +56,7 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption { // Canceling the provided context will stop the background process. func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, opts ...RotatorOption) error { //nolint:gocritic // KeyRotator can only rotate crypto keys. - ctx = dbauthz.AsSystemRestricted(ctx) + ctx = dbauthz.AsKeyRotator(ctx) kr := &rotator{ db: db, logger: logger, diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index f1eb43d2a4ffb..6e54756ce2a86 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -229,7 +229,7 @@ var ( }.WithCachedASTValue() // See cryptokeys package. - subjectCryptoKey = rbac.Subject{ + subjectCryptoKeyRotator = rbac.Subject{ FriendlyName: "Crypto Key Rotator", ID: uuid.Nil.String(), Roles: rbac.Roles([]rbac.Role{ @@ -237,7 +237,25 @@ var ( Identifier: rbac.RoleIdentifier{Name: "keyrotator"}, DisplayName: "Key Rotator", Site: rbac.Permissions(map[string][]policy.Action{ - rbac.ResourceCryptoKey.Type: {policy.WildcardSymbol, policy.ActionRead}, + rbac.ResourceCryptoKey.Type: {policy.WildcardSymbol}, + }), + Org: map[string][]rbac.Permission{}, + User: []rbac.Permission{}, + }, + }), + Scope: rbac.ScopeAll, + }.WithCachedASTValue() + + // See cryptokeys package. + subjectCryptoKeyReader = rbac.Subject{ + FriendlyName: "Crypto Key Reader", + ID: uuid.Nil.String(), + Roles: rbac.Roles([]rbac.Role{ + { + Identifier: rbac.RoleIdentifier{Name: "keyrotator"}, + DisplayName: "Key Rotator", + Site: rbac.Permissions(map[string][]policy.Action{ + rbac.ResourceCryptoKey.Type: {policy.WildcardSymbol}, }), Org: map[string][]rbac.Permission{}, User: []rbac.Permission{}, @@ -301,7 +319,12 @@ func AsHangDetector(ctx context.Context) context.Context { // AsKeyRotator returns a context with an actor that has permissions required for rotating crypto keys. func AsKeyRotator(ctx context.Context) context.Context { - return context.WithValue(ctx, authContextKey{}, subjectCryptoKey) + return context.WithValue(ctx, authContextKey{}, subjectCryptoKeyRotator) +} + +// AsKeyReader returns a context with an actor that has permissions required for reading crypto keys. +func AsKeyReader(ctx context.Context) context.Context { + return context.WithValue(ctx, authContextKey{}, subjectCryptoKeyReader) } // AsSystemRestricted returns a context with an actor that has permissions diff --git a/coderd/workspaceapps/db.go b/coderd/workspaceapps/db.go index 03667e279d42e..2018eba847651 100644 --- a/coderd/workspaceapps/db.go +++ b/coderd/workspaceapps/db.go @@ -38,7 +38,7 @@ type DBTokenProvider struct { DeploymentValues *codersdk.DeploymentValues OAuth2Configs *httpmw.OAuth2Configs WorkspaceAgentInactiveTimeout time.Duration - Signer jwtutils.SigningKeyManager + TokenSigner jwtutils.SigningKeyManager } var _ SignedTokenProvider = &DBTokenProvider{} @@ -56,12 +56,12 @@ func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authoriz DeploymentValues: cfg, OAuth2Configs: oauth2Cfgs, WorkspaceAgentInactiveTimeout: workspaceAgentInactiveTimeout, - Signer: signer, + TokenSigner: signer, } } func (p *DBTokenProvider) FromRequest(r *http.Request) (*SignedToken, bool) { - return FromRequest(r, p.Signer) + return FromRequest(r, p.TokenSigner) } func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *http.Request, issueReq IssueTokenRequest) (*SignedToken, string, bool) { @@ -217,7 +217,7 @@ func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r * token.Claims = jwt.Claims{ Expiry: jwt.NewNumericDate(time.Now().Add(DefaultTokenExpiry)), } - tokenStr, err := jwtutils.Sign(ctx, p.Signer, token) + tokenStr, err := jwtutils.Sign(ctx, p.TokenSigner, token) if err != nil { WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "generate token") return nil, "", false diff --git a/tailnet/resume.go b/tailnet/resume.go index 799c049885b10..6b04632944775 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -21,15 +21,8 @@ import ( const ( DefaultResumeTokenExpiry = 24 * time.Hour - - resumeTokenSigningAlgorithm = jose.HS512 ) -// resumeTokenSigningKeyID is a fixed key ID for the resume token signing key. -// If/when we add support for multiple keys (e.g. key rotation), this will move -// to the database instead. -var resumeTokenSigningKeyID = uuid.MustParse("97166747-9309-4d7f-9071-a230e257c2a4") - // NewInsecureTestResumeTokenProvider returns a ResumeTokenProvider that uses a // random key with short expiry for testing purposes. If any errors occur while // generating the key, the function panics. @@ -40,7 +33,7 @@ func NewInsecureTestResumeTokenProvider() ResumeTokenProvider { } return NewResumeTokenKeyProvider(jwtutils.StaticKeyManager{ ID: uuid.New().String(), - Key: key, + Key: key[:], }, quartz.NewReal(), time.Hour) } From 2ea31abd1d14d81d00b9a00b6468544185437ee3 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 14 Oct 2024 23:43:44 +0000 Subject: [PATCH 08/39] fixing tests --- tailnet/resume.go | 66 ++++------------------- tailnet/resume_test.go | 115 ++++------------------------------------- 2 files changed, 19 insertions(+), 162 deletions(-) diff --git a/tailnet/resume.go b/tailnet/resume.go index 6b04632944775..c2a181414764d 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -3,11 +3,8 @@ package tailnet import ( "context" "crypto/rand" - "database/sql" - "encoding/hex" "time" - "github.com/go-jose/go-jose/v4" "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" "golang.org/x/xerrors" @@ -53,47 +50,6 @@ func GenerateResumeTokenSigningKey() (ResumeTokenSigningKey, error) { return key, nil } -type ResumeTokenSigningKeyDatabaseStore interface { - GetCoordinatorResumeTokenSigningKey(ctx context.Context) (string, error) - UpsertCoordinatorResumeTokenSigningKey(ctx context.Context, key string) error -} - -// ResumeTokenSigningKeyFromDatabase retrieves the coordinator resume token -// signing key from the database. If the key is not found, a new key is -// generated and inserted into the database. -func ResumeTokenSigningKeyFromDatabase(ctx context.Context, db ResumeTokenSigningKeyDatabaseStore) (ResumeTokenSigningKey, error) { - var resumeTokenKey ResumeTokenSigningKey - resumeTokenKeyStr, err := db.GetCoordinatorResumeTokenSigningKey(ctx) - if err != nil && !xerrors.Is(err, sql.ErrNoRows) { - return resumeTokenKey, xerrors.Errorf("get coordinator resume token key: %w", err) - } - if decoded, err := hex.DecodeString(resumeTokenKeyStr); err != nil || len(decoded) != len(resumeTokenKey) { - newKey, err := GenerateResumeTokenSigningKey() - if err != nil { - return resumeTokenKey, xerrors.Errorf("generate fresh coordinator resume token key: %w", err) - } - - resumeTokenKeyStr = hex.EncodeToString(newKey[:]) - err = db.UpsertCoordinatorResumeTokenSigningKey(ctx, resumeTokenKeyStr) - if err != nil { - return resumeTokenKey, xerrors.Errorf("insert freshly generated coordinator resume token key to database: %w", err) - } - } - - resumeTokenKeyBytes, err := hex.DecodeString(resumeTokenKeyStr) - if err != nil { - return resumeTokenKey, xerrors.Errorf("decode coordinator resume token key from database: %w", err) - } - if len(resumeTokenKeyBytes) != len(resumeTokenKey) { - return resumeTokenKey, xerrors.Errorf("coordinator resume token key in database is not the correct length, expect %d got %d", len(resumeTokenKey), len(resumeTokenKeyBytes)) - } - copy(resumeTokenKey[:], resumeTokenKeyBytes) - if resumeTokenKey == [64]byte{} { - return resumeTokenKey, xerrors.Errorf("coordinator resume token key in database is empty") - } - return resumeTokenKey, nil -} - type ResumeTokenKeyProvider struct { key jwtutils.SigningKeyManager clock quartz.Clock @@ -111,19 +67,11 @@ func NewResumeTokenKeyProvider(key jwtutils.SigningKeyManager, clock quartz.Cloc } } -type resumeTokenPayload struct { - jwt.Claims - PeerID uuid.UUID `json:"sub"` - Expiry int64 `json:"exp"` -} - func (p ResumeTokenKeyProvider) GenerateResumeToken(ctx context.Context, peerID uuid.UUID) (*proto.RefreshResumeTokenResponse, error) { exp := p.clock.Now().Add(p.expiry) - payload := resumeTokenPayload{ - PeerID: peerID, - Claims: jwt.Claims{ - Expiry: jwt.NewNumericDate(exp), - }, + payload := jwt.Claims{ + Subject: peerID.String(), + Expiry: jwt.NewNumericDate(exp), } token, err := jwtutils.Sign(ctx, p.key, payload) @@ -142,12 +90,16 @@ func (p ResumeTokenKeyProvider) GenerateResumeToken(ctx context.Context, peerID // returns the payload. If the token is invalid or expired, an error is // returned. func (p ResumeTokenKeyProvider) VerifyResumeToken(ctx context.Context, str string) (uuid.UUID, error) { - var tok resumeTokenPayload + var tok jwt.Claims err := jwtutils.Verify(ctx, p.key, str, &tok, jwtutils.WithVerifyExpected(jwt.Expected{ Time: p.clock.Now(), })) if err != nil { return uuid.Nil, xerrors.Errorf("verify payload: %w", err) } - return tok.PeerID, nil + parsed, err := uuid.Parse(tok.Subject) + if err != nil { + return uuid.Nil, xerrors.Errorf("parse peerID from token: %w", err) + } + return parsed, nil } diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index da67f9a0aa6d2..b0c06f9b66819 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -1,118 +1,20 @@ package tailnet_test import ( - "context" - "encoding/hex" "testing" "time" + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" - "github.com/coder/coder/v2/coderd/database/dbmock" - "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" ) -func TestResumeTokenSigningKeyFromDatabase(t *testing.T) { - t.Parallel() - - assertRandomKey := func(t *testing.T, key tailnet.ResumeTokenSigningKey) { - t.Helper() - assert.NotEqual(t, tailnet.ResumeTokenSigningKey{}, key, "key should not be empty") - assert.NotEqualValues(t, [64]byte{1}, key, "key should not be all 1s") - } - - t.Run("GenerateRetrieve", func(t *testing.T) { - t.Parallel() - - db, _ := dbtestutil.NewDB(t) - ctx := testutil.Context(t, testutil.WaitShort) - key1, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.NoError(t, err) - assertRandomKey(t, key1) - - key2, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.NoError(t, err) - require.Equal(t, key1, key2, "keys should not be different") - }) - - t.Run("GetError", func(t *testing.T) { - t.Parallel() - - db := dbmock.NewMockStore(gomock.NewController(t)) - db.EXPECT().GetCoordinatorResumeTokenSigningKey(gomock.Any()).Return("", assert.AnError) - - ctx := testutil.Context(t, testutil.WaitShort) - _, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.ErrorIs(t, err, assert.AnError) - }) - - t.Run("UpsertError", func(t *testing.T) { - t.Parallel() - - db := dbmock.NewMockStore(gomock.NewController(t)) - db.EXPECT().GetCoordinatorResumeTokenSigningKey(gomock.Any()).Return("", nil) - db.EXPECT().UpsertCoordinatorResumeTokenSigningKey(gomock.Any(), gomock.Any()).Return(assert.AnError) - - ctx := testutil.Context(t, testutil.WaitShort) - _, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.ErrorIs(t, err, assert.AnError) - }) - - t.Run("DecodeErrorShouldRegenerate", func(t *testing.T) { - t.Parallel() - - db := dbmock.NewMockStore(gomock.NewController(t)) - db.EXPECT().GetCoordinatorResumeTokenSigningKey(gomock.Any()).Return("invalid", nil) - - var storedKey tailnet.ResumeTokenSigningKey - db.EXPECT().UpsertCoordinatorResumeTokenSigningKey(gomock.Any(), gomock.Any()).Do(func(_ context.Context, value string) error { - keyBytes, err := hex.DecodeString(value) - require.NoError(t, err) - require.Len(t, keyBytes, len(storedKey)) - copy(storedKey[:], keyBytes) - return nil - }) - - ctx := testutil.Context(t, testutil.WaitShort) - key, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.NoError(t, err) - assertRandomKey(t, key) - require.Equal(t, storedKey, key, "key should match stored value") - }) - - t.Run("LengthErrorShouldRegenerate", func(t *testing.T) { - t.Parallel() - - db := dbmock.NewMockStore(gomock.NewController(t)) - db.EXPECT().GetCoordinatorResumeTokenSigningKey(gomock.Any()).Return("deadbeef", nil) - db.EXPECT().UpsertCoordinatorResumeTokenSigningKey(gomock.Any(), gomock.Any()).Return(nil) - - ctx := testutil.Context(t, testutil.WaitShort) - key, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.NoError(t, err) - assertRandomKey(t, key) - }) - - t.Run("EmptyError", func(t *testing.T) { - t.Parallel() - - db := dbmock.NewMockStore(gomock.NewController(t)) - emptyKey := hex.EncodeToString(make([]byte, 64)) - db.EXPECT().GetCoordinatorResumeTokenSigningKey(gomock.Any()).Return(emptyKey, nil) - - ctx := testutil.Context(t, testutil.WaitShort) - _, err := tailnet.ResumeTokenSigningKeyFromDatabase(ctx, db) - require.ErrorContains(t, err, "is empty") - }) -} - func TestResumeTokenKeyProvider(t *testing.T) { t.Parallel() @@ -156,7 +58,7 @@ func TestResumeTokenKeyProvider(t *testing.T) { _ = clock.Advance(tailnet.DefaultResumeTokenExpiry + time.Second) _, err = provider.VerifyResumeToken(ctx, token.Token) - require.ErrorContains(t, err, "expired") + require.ErrorIs(t, err, jwt.ErrExpired) }) t.Run("InvalidToken", func(t *testing.T) { @@ -175,17 +77,20 @@ func TestResumeTokenKeyProvider(t *testing.T) { // Generate a resume token with a different key otherKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - otherProvider := tailnet.NewResumeTokenKeyProvider(newKeySigner(otherKey), quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) + otherSigner := newKeySigner(otherKey) + otherProvider := tailnet.NewResumeTokenKeyProvider(otherSigner, quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) token, err := otherProvider.GenerateResumeToken(ctx, uuid.New()) require.NoError(t, err) - provider := tailnet.NewResumeTokenKeyProvider(newKeySigner(key), quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) + signer := newKeySigner(key) + signer.ID = otherSigner.ID + provider := tailnet.NewResumeTokenKeyProvider(signer, quartz.NewMock(t), tailnet.DefaultResumeTokenExpiry) _, err = provider.VerifyResumeToken(ctx, token.Token) - require.ErrorContains(t, err, "verify JWS") + require.ErrorIs(t, err, jose.ErrCryptoFailure) }) } -func newKeySigner(key tailnet.ResumeTokenSigningKey) jwtutils.SigningKeyManager { +func newKeySigner(key tailnet.ResumeTokenSigningKey) jwtutils.StaticKeyManager { return jwtutils.StaticKeyManager{ ID: uuid.New().String(), Key: key[:], From d0d168b6af835d0ce6b6e057068bd28bb3438f30 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 15 Oct 2024 00:49:52 +0000 Subject: [PATCH 09/39] time to fix it --- codersdk/deployment.go | 8 +++++--- enterprise/coderd/workspaceproxy.go | 19 ++++++++++++++++++- enterprise/coderd/workspaceproxy_test.go | 5 ++--- enterprise/wsproxy/wsproxy.go | 1 + enterprise/wsproxy/wsproxysdk/wsproxysdk.go | 11 ++--------- tailnet/resume.go | 2 +- tailnet/resume_test.go | 5 +++-- 7 files changed, 32 insertions(+), 19 deletions(-) diff --git a/codersdk/deployment.go b/codersdk/deployment.go index d6840df504b85..44cffd35cf6b0 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -3109,9 +3109,11 @@ func (c *Client) SSHConfiguration(ctx context.Context) (SSHConfigResponse, error type CryptoKeyFeature string const ( - CryptoKeyFeatureWorkspaceApp CryptoKeyFeature = "workspace_apps" - CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" - CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" + CryptoKeyFeatureWorkspaceAppAPIKey CryptoKeyFeature = "workspace_apps_api_key" + //nolint:gosec // This denotes a type of key, not a literal. + CryptoKeyFeatureWorkspaceAppsToken CryptoKeyFeature = "workspace_apps_token" + CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" + CryptoKeyFeatureTailnetResume CryptoKeyFeature = "tailnet_resume" ) type CryptoKey struct { diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index f69046b33c78a..b7411d9922220 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "net/url" + "slices" "strings" "time" @@ -720,13 +721,29 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request) // @Security CoderSessionToken // @Produce json // @Tags Enterprise +// @Param feature query string true "Feature key" // @Success 200 {object} wsproxysdk.CryptoKeysResponse // @Router /workspaceproxies/me/crypto-keys [get] // @x-apidocgen {"skip": true} func (api *API) workspaceProxyCryptoKeys(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() - keys, err := api.Database.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureWorkspaceAppsAPIKey) + feature := database.CryptoKeyFeature(r.URL.Query().Get("feature")) + if feature == "" { + httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ + Message: "Missing feature query parameter.", + }) + return + } + + if !slices.Contains(database.AllCryptoKeyFeatureValues(), feature) { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: fmt.Sprintf("Invalid feature: %q", feature), + }) + return + } + + keys, err := api.Database.GetCryptoKeysByFeature(ctx, feature) if err != nil { httpapi.InternalServerError(rw, err) return diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index ee871c175565d..6fa49052f6973 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -320,7 +320,6 @@ func TestProxyRegisterDeregister(t *testing.T) { } registerRes1, err := proxyClient.RegisterWorkspaceProxy(ctx, req) require.NoError(t, err) - require.NotEmpty(t, registerRes1.AppSecurityKey) require.NotEmpty(t, registerRes1.DERPMeshKey) require.EqualValues(t, 10001, registerRes1.DERPRegionID) require.Empty(t, registerRes1.SiblingReplicas) @@ -955,7 +954,7 @@ func TestGetCryptoKeys(t *testing.T) { Name: testutil.GetRandomName(t), }) - keys, err := proxy.SDKClient.CryptoKeys(ctx) + keys, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppAPIKey) require.NoError(t, err) require.NotEmpty(t, keys) require.Equal(t, 2, len(keys.CryptoKeys)) @@ -987,7 +986,7 @@ func TestGetCryptoKeys(t *testing.T) { client := wsproxysdk.New(cclient.URL) client.SetSessionToken(cclient.SessionToken()) - _, err := client.CryptoKeys(ctx) + _, err := client.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppAPIKey) require.Error(t, err) var sdkErr *codersdk.Error require.ErrorAs(t, err, &sdkErr) diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index 676ddfc25ef3b..27ac1b6cfd2c2 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -198,6 +198,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) { derpServer := derp.NewServer(key.NewNode(), tailnet.Logger(opts.Logger.Named("net.derp"))) ctx, cancel := context.WithCancel(context.Background()) + r := chi.NewRouter() s := &Server{ Options: opts, diff --git a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go index 77d36561c6de8..ee801c0d72f64 100644 --- a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go +++ b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go @@ -205,7 +205,6 @@ type RegisterWorkspaceProxyRequest struct { } type RegisterWorkspaceProxyResponse struct { - AppSecurityKey string `json:"app_security_key"` DERPMeshKey string `json:"derp_mesh_key"` DERPRegionID int32 `json:"derp_region_id"` DERPMap *tailcfg.DERPMap `json:"derp_map"` @@ -372,12 +371,6 @@ func (l *RegisterWorkspaceProxyLoop) Start(ctx context.Context) (RegisterWorkspa } failedAttempts = 0 - // Check for consistency. - if originalRes.AppSecurityKey != resp.AppSecurityKey { - l.failureFn(xerrors.New("app security key has changed, proxy must be restarted")) - return - } - if originalRes.DERPMeshKey != resp.DERPMeshKey { l.failureFn(xerrors.New("DERP mesh key has changed, proxy must be restarted")) return @@ -586,10 +579,10 @@ type CryptoKeysResponse struct { CryptoKeys []codersdk.CryptoKey `json:"crypto_keys"` } -func (c *Client) CryptoKeys(ctx context.Context) (CryptoKeysResponse, error) { +func (c *Client) CryptoKeys(ctx context.Context, feature codersdk.CryptoKeyFeature) (CryptoKeysResponse, error) { res, err := c.Request(ctx, http.MethodGet, "/api/v2/workspaceproxies/me/crypto-keys", - nil, + codersdk.WithQueryParam("feature", string(feature)), ) if err != nil { return CryptoKeysResponse{}, xerrors.Errorf("make request: %w", err) diff --git a/tailnet/resume.go b/tailnet/resume.go index c2a181414764d..b79850bfb6a63 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -63,7 +63,7 @@ func NewResumeTokenKeyProvider(key jwtutils.SigningKeyManager, clock quartz.Cloc return ResumeTokenKeyProvider{ key: key, clock: clock, - expiry: DefaultResumeTokenExpiry, + expiry: expiry, } } diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index b0c06f9b66819..e896c49dfd8f0 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -54,10 +54,11 @@ func TestResumeTokenKeyProvider(t *testing.T) { require.Equal(t, tailnet.DefaultResumeTokenExpiry/2, token.RefreshIn.AsDuration()) require.WithinDuration(t, clock.Now().Add(tailnet.DefaultResumeTokenExpiry), token.ExpiresAt.AsTime(), time.Second) - // Advance time past expiry - _ = clock.Advance(tailnet.DefaultResumeTokenExpiry + time.Second) + // Advance time past expiry. Account for leeway. + _ = clock.Advance(tailnet.DefaultResumeTokenExpiry + time.Second*61) _, err = provider.VerifyResumeToken(ctx, token.Token) + require.Error(t, err) require.ErrorIs(t, err, jwt.ErrExpired) }) From 33cdb96baa0f67e0b4d2f7f34326a436b8cc0f9c Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 16 Oct 2024 20:14:37 +0000 Subject: [PATCH 10/39] Refactor cryptokeys Fetcher to include feature param Enhances flexibility by making the `Fetcher` interface receive a `CryptoKeyFeature` parameter. This change aligns various call sites that implement or utilize `Fetcher`, allowing for more granular queries. --- cli/server.go | 13 +++++- coderd/coderd.go | 52 ++++++++++++++++++++---- coderd/coderdtest/coderdtest.go | 11 ----- coderd/cryptokeys/cache.go | 13 +++--- coderd/cryptokeys/cache_test.go | 2 +- coderd/jwtutils/jwt_test.go | 6 +-- codersdk/deployment.go | 2 +- enterprise/coderd/workspaceproxy_test.go | 4 +- enterprise/wsproxy/keyfetcher.go | 7 ++-- 9 files changed, 72 insertions(+), 38 deletions(-) diff --git a/cli/server.go b/cli/server.go index 10e105b6ee330..3c6c14e300f65 100644 --- a/cli/server.go +++ b/cli/server.go @@ -746,9 +746,18 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("set deployment id: %w", err) } - resumeKeycache, err := cryptokeys.NewSigningCache(logger, options.Database, database.CryptoKeyFeatureTailnetResume) + fetcher := &cryptokeys.DBFetcher{ + DB: options.Database, + } + + // TODO(JonA): The instantiation of this cache + coordinator seems like it should be done inside coderd so that it uses the correct context. + resumeKeycache, err := cryptokeys.NewSigningCache(ctx, + logger, + fetcher, + codersdk.CryptoKeyFeatureTailnetResume, + ) if err != nil { - return xerrors.Errorf("create resume token key cache: %w", err) + return xerrors.Errorf("create tailnet resume key cache: %w", err) } options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeKeycache, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry) diff --git a/coderd/coderd.go b/coderd/coderd.go index 1aa12443cf70f..6a3a0726fa6ee 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -448,6 +448,44 @@ func New(options *Options) *API { if err != nil { panic(xerrors.Errorf("get deployment ID: %w", err)) } + + // Start a background process that rotates keys. + err = cryptokeys.StartRotator(ctx, options.Logger.Named("keyrotator"), options.Database) + if err != nil { + options.Logger.Fatal(ctx, "start key rotator", slog.Error(err)) + } + + fetcher := &cryptokeys.DBFetcher{ + DB: options.Database, + } + + if options.OIDCConvertKeyCache == nil { + options.OIDCConvertKeyCache, err = cryptokeys.NewSigningCache(ctx, + options.Logger.Named("oidc_convert_keycache"), + fetcher, + codersdk.CryptoKeyFeatureOIDCConvert, + ) + must(options.Logger, "start oidc convert key cache", err) + } + + if options.AppSigningKeyCache == nil { + options.AppSigningKeyCache, err = cryptokeys.NewSigningCache(ctx, + options.Logger.Named("app_signing_keycache"), + fetcher, + codersdk.CryptoKeyFeatureWorkspaceAppsToken, + ) + must(options.Logger, "start app signing key cache", err) + } + + if options.AppEncryptionKeyCache == nil { + options.AppEncryptionKeyCache, err = cryptokeys.NewEncryptionCache(ctx, + options.Logger.Named("app_encryption_keycache"), + fetcher, + codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey, + ) + must(options.Logger, "start app encryption key cache", err) + } + api := &API{ ctx: ctx, cancel: cancel, @@ -484,7 +522,7 @@ func New(options *Options) *API { options.Database, options.Pubsub, ), - dbRolluper: options.DatabaseRolluper, + dbRolluper: options.DatabaseRolluper, } f := appearance.NewDefaultFetcher(api.DeploymentValues.DocsURL.String()) @@ -613,12 +651,6 @@ func New(options *Options) *API { api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err)) } - // Start a background process that rotates keys. - err = cryptokeys.StartRotator(api.ctx, api.Logger.Named("keyrotator"), api.Database) - if err != nil { - api.Logger.Fatal(api.ctx, "start key rotator", slog.Error(err)) - } - api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ Database: options.Database, Logger: options.Logger.Named("workspacestats"), @@ -1612,3 +1644,9 @@ func ReadExperiments(log slog.Logger, raw []string) codersdk.Experiments { } return exps } + +func must(logger slog.Logger, msg string, err error) { + if err != nil { + logger.Fatal(context.Background(), msg, slog.Error(err)) + } +} diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 1ea15ea3ddce4..cc6611d766faf 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -55,7 +55,6 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/autobuild" "github.com/coder/coder/v2/coderd/awsidentity" - "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbauthz" @@ -326,13 +325,6 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can } auditor.Store(&options.Auditor) - oidcConvertKeyCache, err := cryptokeys.NewSigningCache(options.Logger.Named("oidc_convert_keycache"), options.Database, database.CryptoKeyFeatureOIDCConvert) - require.NoError(t, err) - appSigningKeyCache, err := cryptokeys.NewSigningCache(options.Logger.Named("app_signing_keycache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsToken) - require.NoError(t, err) - appEncryptionKeyCache, err := cryptokeys.NewEncryptionCache(options.Logger.Named("app_encryption_keycache"), options.Database, database.CryptoKeyFeatureWorkspaceAppsAPIKey) - require.NoError(t, err) - ctx, cancelFunc := context.WithCancel(context.Background()) lifecycleExecutor := autobuild.NewExecutor( ctx, @@ -540,9 +532,6 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can WorkspaceUsageTracker: wuTracker, NotificationsEnqueuer: options.NotificationsEnqueuer, OneTimePasscodeValidityPeriod: options.OneTimePasscodeValidityPeriod, - AppSigningKeyCache: appSigningKeyCache, - AppEncryptionKeyCache: appEncryptionKeyCache, - OIDCConvertKeyCache: oidcConvertKeyCache, } } diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 74fb025d416fd..bf268abea7705 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -25,7 +25,7 @@ var ( ) type Fetcher interface { - Fetch(ctx context.Context) ([]codersdk.CryptoKey, error) + Fetch(ctx context.Context, feature codersdk.CryptoKeyFeature) ([]codersdk.CryptoKey, error) } type EncryptionKeycache interface { @@ -62,12 +62,11 @@ const ( ) type DBFetcher struct { - DB database.Store - Feature database.CryptoKeyFeature + DB database.Store } -func (d *DBFetcher) Fetch(ctx context.Context) ([]codersdk.CryptoKey, error) { - keys, err := d.DB.GetCryptoKeysByFeature(ctx, d.Feature) +func (d *DBFetcher) Fetch(ctx context.Context, feature codersdk.CryptoKeyFeature) ([]codersdk.CryptoKey, error) { + keys, err := d.DB.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeature(feature)) if err != nil { return nil, xerrors.Errorf("get crypto keys by feature: %w", err) } @@ -198,7 +197,7 @@ func (c *cache) VerifyingKey(ctx context.Context, id string) (interface{}, error } func isEncryptionKeyFeature(feature codersdk.CryptoKeyFeature) bool { - return feature == codersdk.CryptoKeyFeatureWorkspaceApp + return feature == codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey } func isSigningKeyFeature(feature codersdk.CryptoKeyFeature) bool { @@ -332,7 +331,7 @@ func (c *cache) refresh() { // cryptoKeys queries the control plane for the crypto keys. // Outside of initialization, this should only be called by fetch. func (c *cache) cryptoKeys(ctx context.Context) (map[int32]codersdk.CryptoKey, error) { - keys, err := c.fetcher.Fetch(ctx) + keys, err := c.fetcher.Fetch(ctx, c.feature) if err != nil { return nil, xerrors.Errorf("crypto keys: %w", err) } diff --git a/coderd/cryptokeys/cache_test.go b/coderd/cryptokeys/cache_test.go index 92fc4527ae7b3..cda87315605a4 100644 --- a/coderd/cryptokeys/cache_test.go +++ b/coderd/cryptokeys/cache_test.go @@ -488,7 +488,7 @@ type fakeFetcher struct { called int } -func (f *fakeFetcher) Fetch(_ context.Context) ([]codersdk.CryptoKey, error) { +func (f *fakeFetcher) Fetch(_ context.Context, _ codersdk.CryptoKeyFeature) ([]codersdk.CryptoKey, error) { f.called++ return f.keys, nil } diff --git a/coderd/jwtutils/jwt_test.go b/coderd/jwtutils/jwt_test.go index 4c57e0451fcb6..5d1f4d48bdb4a 100644 --- a/coderd/jwtutils/jwt_test.go +++ b/coderd/jwtutils/jwt_test.go @@ -240,7 +240,7 @@ func TestJWS(t *testing.T) { StartsAt: time.Now(), }) log = slogtest.Make(t, nil) - fetcher = &cryptokeys.DBFetcher{DB: db, Feature: database.CryptoKeyFeatureOidcConvert} + fetcher = &cryptokeys.DBFetcher{DB: db} ) cache, err := cryptokeys.NewSigningCache(ctx, log, fetcher, codersdk.CryptoKeyFeatureOIDCConvert) @@ -331,10 +331,10 @@ func TestJWE(t *testing.T) { }) log = slogtest.Make(t, nil) - fetcher = &cryptokeys.DBFetcher{DB: db, Feature: database.CryptoKeyFeatureWorkspaceApps} + fetcher = &cryptokeys.DBFetcher{DB: db} ) - cache, err := cryptokeys.NewEncryptionCache(ctx, log, fetcher, codersdk.CryptoKeyFeatureWorkspaceApp) + cache, err := cryptokeys.NewEncryptionCache(ctx, log, fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) claims := testClaims{ diff --git a/codersdk/deployment.go b/codersdk/deployment.go index 44cffd35cf6b0..391d0039f0369 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -3109,7 +3109,7 @@ func (c *Client) SSHConfiguration(ctx context.Context) (SSHConfigResponse, error type CryptoKeyFeature string const ( - CryptoKeyFeatureWorkspaceAppAPIKey CryptoKeyFeature = "workspace_apps_api_key" + CryptoKeyFeatureWorkspaceAppsAPIKey CryptoKeyFeature = "workspace_apps_api_key" //nolint:gosec // This denotes a type of key, not a literal. CryptoKeyFeatureWorkspaceAppsToken CryptoKeyFeature = "workspace_apps_token" CryptoKeyFeatureOIDCConvert CryptoKeyFeature = "oidc_convert" diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 6fa49052f6973..ad6bf017ea2af 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -954,7 +954,7 @@ func TestGetCryptoKeys(t *testing.T) { Name: testutil.GetRandomName(t), }) - keys, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppAPIKey) + keys, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) require.NotEmpty(t, keys) require.Equal(t, 2, len(keys.CryptoKeys)) @@ -986,7 +986,7 @@ func TestGetCryptoKeys(t *testing.T) { client := wsproxysdk.New(cclient.URL) client.SetSessionToken(cclient.SessionToken()) - _, err := client.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppAPIKey) + _, err := client.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) require.Error(t, err) var sdkErr *codersdk.Error require.ErrorAs(t, err, &sdkErr) diff --git a/enterprise/wsproxy/keyfetcher.go b/enterprise/wsproxy/keyfetcher.go index 81b71301b610f..007a4bd5f944b 100644 --- a/enterprise/wsproxy/keyfetcher.go +++ b/enterprise/wsproxy/keyfetcher.go @@ -12,12 +12,11 @@ import ( var _ cryptokeys.Fetcher = &ProxyFetcher{} type ProxyFetcher struct { - Client *wsproxysdk.Client - Feature codersdk.CryptoKeyFeature + Client *wsproxysdk.Client } -func (p *ProxyFetcher) Fetch(ctx context.Context) ([]codersdk.CryptoKey, error) { - keys, err := p.Client.CryptoKeys(ctx) +func (p *ProxyFetcher) Fetch(ctx context.Context, feature codersdk.CryptoKeyFeature) ([]codersdk.CryptoKey, error) { + keys, err := p.Client.CryptoKeys(ctx, feature) if err != nil { return nil, xerrors.Errorf("crypto keys: %w", err) } From 08570b76f9ca9f1660b817b503545286c021f52d Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 16 Oct 2024 20:51:48 +0000 Subject: [PATCH 11/39] Refactor key caching and logging behavior - Improve code clarity by refactoring key caching logic. - Simplify logger initialization for signing and encryption caches. - Ensure consistent closing of caches in the API and server. --- coderd/activitybump_test.go | 2 +- coderd/coderd.go | 11 +++--- coderd/cryptokeys/cache.go | 3 ++ enterprise/coderd/coderdenttest/proxytest.go | 6 +++- enterprise/wsproxy/keyfetcher.go | 3 +- enterprise/wsproxy/wsproxy.go | 35 ++++++++++++++++---- 6 files changed, 46 insertions(+), 14 deletions(-) diff --git a/coderd/activitybump_test.go b/coderd/activitybump_test.go index 2f4413dff9fee..60aec23475885 100644 --- a/coderd/activitybump_test.go +++ b/coderd/activitybump_test.go @@ -125,7 +125,7 @@ func TestWorkspaceActivityBump(t *testing.T) { } // maxTimeDrift is how long we are willing wait for a deadline to - // be increased. Since it could have been bumped at the intial + // be increased. Since it could have been bumped at the initial maxTimeDrift := testutil.WaitMedium updatedAfter := dbtime.Now() diff --git a/coderd/coderd.go b/coderd/coderd.go index 6a3a0726fa6ee..aca3da5c739ac 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -461,7 +461,7 @@ func New(options *Options) *API { if options.OIDCConvertKeyCache == nil { options.OIDCConvertKeyCache, err = cryptokeys.NewSigningCache(ctx, - options.Logger.Named("oidc_convert_keycache"), + options.Logger, fetcher, codersdk.CryptoKeyFeatureOIDCConvert, ) @@ -470,7 +470,7 @@ func New(options *Options) *API { if options.AppSigningKeyCache == nil { options.AppSigningKeyCache, err = cryptokeys.NewSigningCache(ctx, - options.Logger.Named("app_signing_keycache"), + options.Logger, fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsToken, ) @@ -479,7 +479,7 @@ func New(options *Options) *API { if options.AppEncryptionKeyCache == nil { options.AppEncryptionKeyCache, err = cryptokeys.NewEncryptionCache(ctx, - options.Logger.Named("app_encryption_keycache"), + options.Logger, fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey, ) @@ -522,7 +522,7 @@ func New(options *Options) *API { options.Database, options.Pubsub, ), - dbRolluper: options.DatabaseRolluper, + dbRolluper: options.DatabaseRolluper, } f := appearance.NewDefaultFetcher(api.DeploymentValues.DocsURL.String()) @@ -1474,6 +1474,9 @@ func (api *API) Close() error { _ = api.agentProvider.Close() _ = api.statsReporter.Close() _ = api.NetworkTelemetryBatcher.Close() + _ = api.OIDCConvertKeyCache.Close() + _ = api.AppSigningKeyCache.Close() + _ = api.AppEncryptionKeyCache.Close() return nil } diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index bf268abea7705..5651de07cb413 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -3,6 +3,7 @@ package cryptokeys import ( "context" "encoding/hex" + "fmt" "io" "strconv" "sync" @@ -108,6 +109,7 @@ func NewSigningCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, if !isSigningKeyFeature(feature) { return nil, xerrors.Errorf("invalid feature: %s", feature) } + logger = logger.Named(fmt.Sprintf("%s_signing_keycache", feature)) return newCache(ctx, logger, fetcher, feature, opts...) } @@ -117,6 +119,7 @@ func NewEncryptionCache(ctx context.Context, logger slog.Logger, fetcher Fetcher if !isEncryptionKeyFeature(feature) { return nil, xerrors.Errorf("invalid feature: %s", feature) } + logger = logger.Named(fmt.Sprintf("%s_encryption_keycache", feature)) return newCache(ctx, logger, fetcher, feature, opts...) } diff --git a/enterprise/coderd/coderdenttest/proxytest.go b/enterprise/coderd/coderdenttest/proxytest.go index 6e5a822bdf251..a6f2c7384b16f 100644 --- a/enterprise/coderd/coderdenttest/proxytest.go +++ b/enterprise/coderd/coderdenttest/proxytest.go @@ -65,6 +65,8 @@ type WorkspaceProxy struct { // owner client. If a token is provided, the proxy will become a replica of the // existing proxy region. func NewWorkspaceProxyReplica(t *testing.T, coderdAPI *coderd.API, owner *codersdk.Client, options *ProxyOptions) WorkspaceProxy { + t.Helper() + ctx, cancelFunc := context.WithCancel(context.Background()) t.Cleanup(cancelFunc) @@ -142,8 +144,10 @@ func NewWorkspaceProxyReplica(t *testing.T, coderdAPI *coderd.API, owner *coders statsCollectorOptions.Flush = options.FlushStats } + logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug).With(slog.F("server_url", serverURL.String())) + wssrv, err := wsproxy.New(ctx, &wsproxy.Options{ - Logger: slogtest.Make(t, nil).Leveled(slog.LevelDebug).With(slog.F("server_url", serverURL.String())), + Logger: logger, Experiments: options.Experiments, DashboardURL: coderdAPI.AccessURL, AccessURL: accessURL, diff --git a/enterprise/wsproxy/keyfetcher.go b/enterprise/wsproxy/keyfetcher.go index 007a4bd5f944b..1a1745d6ccd2d 100644 --- a/enterprise/wsproxy/keyfetcher.go +++ b/enterprise/wsproxy/keyfetcher.go @@ -3,10 +3,11 @@ package wsproxy import ( "context" + "golang.org/x/xerrors" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" - "golang.org/x/xerrors" ) var _ cryptokeys.Fetcher = &ProxyFetcher{} diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index 27ac1b6cfd2c2..a6e7d70ca1a99 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -93,9 +93,7 @@ type Options struct { // from the dashboardURL. This should only be used in development. AllowAllCors bool - StatsCollectorOptions workspaceapps.StatsCollectorOptions - WorkspaceAppsEncryptionKeycache cryptokeys.EncryptionKeycache - WorkspaceAppsSigningKeycache cryptokeys.SigningKeycache + StatsCollectorOptions workspaceapps.StatsCollectorOptions } func (o *Options) Validate() error { @@ -133,6 +131,9 @@ type Server struct { // the moon's token. SDKClient *wsproxysdk.Client + WorkspaceAppsEncryptionKeycache cryptokeys.EncryptionKeycache + WorkspaceAppsSigningKeycache cryptokeys.SigningKeycache + // DERP derpMesh *derpmesh.Mesh derpMeshTLSConfig *tls.Config @@ -199,8 +200,28 @@ func New(ctx context.Context, opts *Options) (*Server, error) { ctx, cancel := context.WithCancel(context.Background()) + encryptionCache, err := cryptokeys.NewEncryptionCache(ctx, + opts.Logger, + &ProxyFetcher{Client: client}, + codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey, + ) + if err != nil { + return nil, xerrors.Errorf("create api key encryption cache: %w", err) + } + signingCache, err := cryptokeys.NewSigningCache(ctx, + opts.Logger, + &ProxyFetcher{Client: client}, + codersdk.CryptoKeyFeatureWorkspaceAppsToken, + ) + if err != nil { + return nil, xerrors.Errorf("create api token signing cache: %w", err) + } + r := chi.NewRouter() s := &Server{ + ctx: ctx, + cancel: cancel, + Options: opts, Handler: r, DashboardURL: opts.DashboardURL, @@ -210,8 +231,6 @@ func New(ctx context.Context, opts *Options) (*Server, error) { SDKClient: client, derpMesh: derpmesh.New(opts.Logger.Named("net.derpmesh"), derpServer, meshTLSConfig), derpMeshTLSConfig: meshTLSConfig, - ctx: ctx, - cancel: cancel, } // Register the workspace proxy with the primary coderd instance and start a @@ -280,8 +299,8 @@ func New(ctx context.Context, opts *Options) (*Server, error) { AccessURL: opts.AccessURL, AppHostname: opts.AppHostname, Client: client, - SigningKey: opts.WorkspaceAppsSigningKeycache, - EncryptingKey: opts.WorkspaceAppsEncryptionKeycache, + SigningKey: signingCache, + EncryptingKey: encryptionCache, Logger: s.Logger.Named("proxy_token_provider"), }, @@ -432,6 +451,8 @@ func (s *Server) Close() error { err = multierror.Append(err, agentProviderErr) } s.SDKClient.SDKClient.HTTPClient.CloseIdleConnections() + _ = s.WorkspaceAppsSigningKeycache.Close() + _ = s.WorkspaceAppsEncryptionKeycache.Close() return err } From b770762064c14d60071dce8efcecf21250b6b58b Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 16 Oct 2024 21:06:20 +0000 Subject: [PATCH 12/39] Refactor crypto_key_feature migration logic This change enhances the crypto_keys table by updating enum handling for features. It introduces distinct roles for key storage, aiding in better distinction and maintenance of key features. --- .../000266_cryptokey_features.down.sql | 18 +++++++++ .../000266_cryptokey_features.up.sql | 18 +++++++++ .../fixtures/000266_cryptokey_features.up.sql | 40 +++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 coderd/database/migrations/000266_cryptokey_features.down.sql create mode 100644 coderd/database/migrations/000266_cryptokey_features.up.sql create mode 100644 coderd/database/migrations/testdata/fixtures/000266_cryptokey_features.up.sql diff --git a/coderd/database/migrations/000266_cryptokey_features.down.sql b/coderd/database/migrations/000266_cryptokey_features.down.sql new file mode 100644 index 0000000000000..7cdd00d222da8 --- /dev/null +++ b/coderd/database/migrations/000266_cryptokey_features.down.sql @@ -0,0 +1,18 @@ +-- Step 1: Remove the new entries from crypto_keys table +DELETE FROM crypto_keys +WHERE feature IN ('workspace_apps_token', 'workspace_apps_api_key'); + +CREATE TYPE old_crypto_key_feature AS ENUM ( + 'workspace_apps', + 'oidc_convert', + 'tailnet_resume' +); + +ALTER TABLE crypto_keys + ALTER COLUMN feature TYPE old_crypto_key_feature + USING (feature::text::old_crypto_key_feature); + +DROP TYPE crypto_key_feature; + +ALTER TYPE old_crypto_key_feature RENAME TO crypto_key_feature; + diff --git a/coderd/database/migrations/000266_cryptokey_features.up.sql b/coderd/database/migrations/000266_cryptokey_features.up.sql new file mode 100644 index 0000000000000..bca75d220d0c7 --- /dev/null +++ b/coderd/database/migrations/000266_cryptokey_features.up.sql @@ -0,0 +1,18 @@ +-- Create a new enum type with the desired values +CREATE TYPE new_crypto_key_feature AS ENUM ( + 'workspace_apps_token', + 'workspace_apps_api_key', + 'oidc_convert', + 'tailnet_resume' +); + +DELETE FROM crypto_keys WHERE feature = 'workspace_apps'; + +-- Drop the old type and rename the new one +ALTER TABLE crypto_keys + ALTER COLUMN feature TYPE new_crypto_key_feature + USING (feature::text::new_crypto_key_feature); + +DROP TYPE crypto_key_feature; + +ALTER TYPE new_crypto_key_feature RENAME TO crypto_key_feature; diff --git a/coderd/database/migrations/testdata/fixtures/000266_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000266_cryptokey_features.up.sql new file mode 100644 index 0000000000000..5cb2cd4c95509 --- /dev/null +++ b/coderd/database/migrations/testdata/fixtures/000266_cryptokey_features.up.sql @@ -0,0 +1,40 @@ +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'workspace_apps_token', + 1, + 'abc', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'workspace_apps_api_key', + 1, + 'def', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'oidc_convert', + 2, + 'ghi', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + +INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) +VALUES ( + 'tailnet_resume', + 2, + 'jkl', + NULL, + '1970-01-01 00:00:00 UTC'::timestamptz, + '2100-01-01 00:00:00 UTC'::timestamptz +); + From 7557ed289b08605f092e0ee305c9c34453df0757 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Wed, 16 Oct 2024 23:25:48 +0000 Subject: [PATCH 13/39] Refactor key management and enhance logging - Improve clarity by naming loggers used in key cache creation. - Adjust key cache context to utilize KeyReader for consistent context handling. - Refactor API to use central key cache management approach. - Enhance error messages for crypto key fetching. - Update test to add safety against unexpected panics. --- coderd/coderd.go | 11 ++--- coderd/cryptokeys/cache.go | 30 ++++++------ coderd/userauth.go | 9 +++- coderd/workspaceapps/proxy.go | 7 ++- coderd/workspaceapps/token.go | 6 --- enterprise/wsproxy/tokenprovider.go | 12 ++--- enterprise/wsproxy/wsproxy.go | 53 ++++++++++++--------- enterprise/wsproxy/wsproxy_test.go | 6 +++ enterprise/wsproxy/wsproxysdk/wsproxysdk.go | 2 +- 9 files changed, 75 insertions(+), 61 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index aca3da5c739ac..b90b40221b560 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -461,7 +461,7 @@ func New(options *Options) *API { if options.OIDCConvertKeyCache == nil { options.OIDCConvertKeyCache, err = cryptokeys.NewSigningCache(ctx, - options.Logger, + options.Logger.Named("oidc_convert_keycache"), fetcher, codersdk.CryptoKeyFeatureOIDCConvert, ) @@ -470,7 +470,7 @@ func New(options *Options) *API { if options.AppSigningKeyCache == nil { options.AppSigningKeyCache, err = cryptokeys.NewSigningCache(ctx, - options.Logger, + options.Logger.Named("app_signing_keycache"), fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsToken, ) @@ -683,10 +683,9 @@ func New(options *Options) *API { AgentProvider: api.agentProvider, StatsCollector: workspaceapps.NewStatsCollector(options.WorkspaceAppsStatsCollectorOptions), - DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), - SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), - Signer: options.AppSigningKeyCache, - EncryptingKeyManager: options.AppEncryptionKeyCache, + DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), + SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), + APIKeyEncryptionKey: options.AppEncryptionKeyCache, } apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{ diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 5651de07cb413..601916fec9419 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -14,6 +14,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" + "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/codersdk" "github.com/coder/quartz" ) @@ -77,12 +78,12 @@ func (d *DBFetcher) Fetch(ctx context.Context, feature codersdk.CryptoKeyFeature // cache implements the caching functionality for both signing and encryption keys. type cache struct { - clock quartz.Clock - refreshCtx context.Context - refreshCancel context.CancelFunc - fetcher Fetcher - logger slog.Logger - feature codersdk.CryptoKeyFeature + ctx context.Context + cancel context.CancelFunc + clock quartz.Clock + fetcher Fetcher + logger slog.Logger + feature codersdk.CryptoKeyFeature mu sync.Mutex keys map[int32]codersdk.CryptoKey @@ -136,12 +137,13 @@ func newCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, feature } cache.cond = sync.NewCond(&cache.mu) - cache.refreshCtx, cache.refreshCancel = context.WithCancel(ctx) + //nolint:gocritic // We need to be able to read the keys in order to cache them. + cache.ctx, cache.cancel = context.WithCancel(dbauthz.AsKeyReader(ctx)) cache.refresher = cache.clock.AfterFunc(refreshInterval, cache.refresh) - keys, err := cache.cryptoKeys(ctx) + keys, err := cache.cryptoKeys(cache.ctx) if err != nil { - cache.refreshCancel() + cache.cancel() return nil, xerrors.Errorf("initial fetch: %w", err) } cache.keys = keys @@ -205,7 +207,7 @@ func isEncryptionKeyFeature(feature codersdk.CryptoKeyFeature) bool { func isSigningKeyFeature(feature codersdk.CryptoKeyFeature) bool { switch feature { - case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert: + case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken: return true default: return false @@ -315,9 +317,9 @@ func (c *cache) refresh() { c.fetching = true c.mu.Unlock() - keys, err := c.cryptoKeys(c.refreshCtx) + keys, err := c.cryptoKeys(c.ctx) if err != nil { - c.logger.Error(c.refreshCtx, "fetch crypto keys", slog.Error(err)) + c.logger.Error(c.ctx, "fetch crypto keys", slog.Error(err)) return } @@ -336,7 +338,7 @@ func (c *cache) refresh() { func (c *cache) cryptoKeys(ctx context.Context) (map[int32]codersdk.CryptoKey, error) { keys, err := c.fetcher.Fetch(ctx, c.feature) if err != nil { - return nil, xerrors.Errorf("crypto keys: %w", err) + return nil, xerrors.Errorf("fetch: %w", err) } cache := toKeyMap(keys, c.clock.Now()) return cache, nil @@ -363,7 +365,7 @@ func (c *cache) Close() error { } c.closed = true - c.refreshCancel() + c.cancel() c.refresher.Stop() c.cond.Broadcast() diff --git a/coderd/userauth.go b/coderd/userauth.go index a702ffb75f79b..f4e74641c71c4 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -167,7 +167,9 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { ToLoginType: req.ToType, } - token, err := jwtutils.Sign(dbauthz.AsKeyRotator(ctx), api.OIDCConvertKeyCache, claims) + //nolint:gocritic // We need to read the system signing key + // in order to sign the token. + token, err := jwtutils.Sign(dbauthz.AsKeyReader(ctx), api.OIDCConvertKeyCache, claims) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error signing state jwt.", @@ -1676,7 +1678,10 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data } } var claims OAuthConvertStateClaims - err = jwtutils.Verify(dbauthz.AsKeyRotator(ctx), api.OIDCConvertKeyCache, jwtCookie.Value, &claims) + + //nolint:gocritic // We need to read the system signing key + // in order to verify the token. + err = jwtutils.Verify(dbauthz.AsKeyReader(ctx), api.OIDCConvertKeyCache, jwtCookie.Value, &claims) if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) { // These errors are probably because the user is mixing 2 coder deployments. return database.User{}, idpsync.HTTPError{ diff --git a/coderd/workspaceapps/proxy.go b/coderd/workspaceapps/proxy.go index bf5e5bd517470..ad6d445e15310 100644 --- a/coderd/workspaceapps/proxy.go +++ b/coderd/workspaceapps/proxy.go @@ -100,9 +100,8 @@ type Server struct { HostnameRegex *regexp.Regexp RealIPConfig *httpmw.RealIPConfig - SignedTokenProvider SignedTokenProvider - Signer jwtutils.SigningKeyManager - EncryptingKeyManager jwtutils.EncryptingKeyManager + SignedTokenProvider SignedTokenProvider + APIKeyEncryptionKey jwtutils.EncryptingKeyManager // DisablePathApps disables path-based apps. This is a security feature as path // based apps share the same cookie as the dashboard, and are susceptible to XSS @@ -181,7 +180,7 @@ func (s *Server) handleAPIKeySmuggling(rw http.ResponseWriter, r *http.Request, // Exchange the encoded API key for a real one. var payload EncryptedAPIKeyPayload - err := jwtutils.Decrypt(ctx, s.EncryptingKeyManager, encryptedAPIKey, &payload, jwtutils.WithDecryptExpected(jwt.Expected{ + err := jwtutils.Decrypt(ctx, s.APIKeyEncryptionKey, encryptedAPIKey, &payload, jwtutils.WithDecryptExpected(jwt.Expected{ Time: time.Now(), })) if err != nil { diff --git a/coderd/workspaceapps/token.go b/coderd/workspaceapps/token.go index 24b7ea9ba580b..ee8101590f88e 100644 --- a/coderd/workspaceapps/token.go +++ b/coderd/workspaceapps/token.go @@ -5,7 +5,6 @@ import ( "strings" "time" - "github.com/go-jose/go-jose/v4" "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" @@ -13,11 +12,6 @@ import ( "github.com/coder/coder/v2/codersdk" ) -const ( - tokenSigningAlgorithm = jose.HS512 - apiKeyEncryptionAlgorithm = jose.A256GCMKW -) - // SignedToken is the struct data contained inside a workspace app JWE. It // contains the details of the workspace app that the token is valid for to // avoid database queries. diff --git a/enterprise/wsproxy/tokenprovider.go b/enterprise/wsproxy/tokenprovider.go index 45f955d3f2dec..8e452f1766aa0 100644 --- a/enterprise/wsproxy/tokenprovider.go +++ b/enterprise/wsproxy/tokenprovider.go @@ -19,14 +19,14 @@ type TokenProvider struct { AccessURL *url.URL AppHostname string - Client *wsproxysdk.Client - SigningKey jwtutils.SigningKeyManager - EncryptingKey jwtutils.EncryptingKeyManager - Logger slog.Logger + Client *wsproxysdk.Client + TokenSigningKey jwtutils.SigningKeyManager + APIKeyEncryptionKey jwtutils.EncryptingKeyManager + Logger slog.Logger } func (p *TokenProvider) FromRequest(r *http.Request) (*workspaceapps.SignedToken, bool) { - return workspaceapps.FromRequest(r, p.SigningKey) + return workspaceapps.FromRequest(r, p.TokenSigningKey) } func (p *TokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *http.Request, issueReq workspaceapps.IssueTokenRequest) (*workspaceapps.SignedToken, string, bool) { @@ -45,7 +45,7 @@ func (p *TokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *ht // Check that it verifies properly and matches the string. var token workspaceapps.SignedToken - err = jwtutils.Verify(ctx, p.SigningKey, resp.SignedTokenStr, &token) + err = jwtutils.Verify(ctx, p.TokenSigningKey, resp.SignedTokenStr, &token) if err != nil { workspaceapps.WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "failed to verify newly generated signed token") return nil, "", false diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index a6e7d70ca1a99..5410572802087 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -131,8 +131,12 @@ type Server struct { // the moon's token. SDKClient *wsproxysdk.Client - WorkspaceAppsEncryptionKeycache cryptokeys.EncryptionKeycache - WorkspaceAppsSigningKeycache cryptokeys.SigningKeycache + // apiKeyEncryptionKeycache manages the encryption keys for smuggling API + // tokens to the alternate domain when using workspace apps. + apiKeyEncryptionKeycache cryptokeys.EncryptionKeycache + // appTokenSigningKeycache manages the signing keys for signing the app + // tokens we use for workspace apps. + appTokenSigningKeycache cryptokeys.SigningKeycache // DERP derpMesh *derpmesh.Mesh @@ -206,6 +210,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) { codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey, ) if err != nil { + cancel() return nil, xerrors.Errorf("create api key encryption cache: %w", err) } signingCache, err := cryptokeys.NewSigningCache(ctx, @@ -214,6 +219,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) { codersdk.CryptoKeyFeatureWorkspaceAppsToken, ) if err != nil { + cancel() return nil, xerrors.Errorf("create api token signing cache: %w", err) } @@ -222,15 +228,17 @@ func New(ctx context.Context, opts *Options) (*Server, error) { ctx: ctx, cancel: cancel, - Options: opts, - Handler: r, - DashboardURL: opts.DashboardURL, - Logger: opts.Logger.Named("net.workspace-proxy"), - TracerProvider: opts.Tracing, - PrometheusRegistry: opts.PrometheusRegistry, - SDKClient: client, - derpMesh: derpmesh.New(opts.Logger.Named("net.derpmesh"), derpServer, meshTLSConfig), - derpMeshTLSConfig: meshTLSConfig, + Options: opts, + Handler: r, + DashboardURL: opts.DashboardURL, + Logger: opts.Logger.Named("net.workspace-proxy"), + TracerProvider: opts.Tracing, + PrometheusRegistry: opts.PrometheusRegistry, + SDKClient: client, + derpMesh: derpmesh.New(opts.Logger.Named("net.derpmesh"), derpServer, meshTLSConfig), + derpMeshTLSConfig: meshTLSConfig, + apiKeyEncryptionKeycache: encryptionCache, + appTokenSigningKeycache: signingCache, } // Register the workspace proxy with the primary coderd instance and start a @@ -295,20 +303,21 @@ func New(ctx context.Context, opts *Options) (*Server, error) { HostnameRegex: opts.AppHostnameRegex, RealIPConfig: opts.RealIPConfig, SignedTokenProvider: &TokenProvider{ - DashboardURL: opts.DashboardURL, - AccessURL: opts.AccessURL, - AppHostname: opts.AppHostname, - Client: client, - SigningKey: signingCache, - EncryptingKey: encryptionCache, - Logger: s.Logger.Named("proxy_token_provider"), + DashboardURL: opts.DashboardURL, + AccessURL: opts.AccessURL, + AppHostname: opts.AppHostname, + Client: client, + TokenSigningKey: signingCache, + APIKeyEncryptionKey: encryptionCache, + Logger: s.Logger.Named("proxy_token_provider"), }, DisablePathApps: opts.DisablePathApps, SecureAuthCookie: opts.SecureAuthCookie, - AgentProvider: agentProvider, - StatsCollector: workspaceapps.NewStatsCollector(opts.StatsCollectorOptions), + AgentProvider: agentProvider, + StatsCollector: workspaceapps.NewStatsCollector(opts.StatsCollectorOptions), + APIKeyEncryptionKey: encryptionCache, } derpHandler := derphttp.Handler(derpServer) @@ -451,8 +460,8 @@ func (s *Server) Close() error { err = multierror.Append(err, agentProviderErr) } s.SDKClient.SDKClient.HTTPClient.CloseIdleConnections() - _ = s.WorkspaceAppsSigningKeycache.Close() - _ = s.WorkspaceAppsEncryptionKeycache.Close() + _ = s.appTokenSigningKeycache.Close() + _ = s.apiKeyEncryptionKeycache.Close() return err } diff --git a/enterprise/wsproxy/wsproxy_test.go b/enterprise/wsproxy/wsproxy_test.go index 3d3926c5afae7..64aca14377318 100644 --- a/enterprise/wsproxy/wsproxy_test.go +++ b/enterprise/wsproxy/wsproxy_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "runtime/debug" "testing" "time" @@ -321,6 +322,11 @@ resourceLoop: func TestDERPEndToEnd(t *testing.T) { t.Parallel() + defer func() { + if r := recover(); r != nil { + debug.PrintStack() + } + }() deploymentValues := coderdtest.DeploymentValues(t) deploymentValues.Experiments = []string{ diff --git a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go index ee801c0d72f64..a8f22c2b93063 100644 --- a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go +++ b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go @@ -581,7 +581,7 @@ type CryptoKeysResponse struct { func (c *Client) CryptoKeys(ctx context.Context, feature codersdk.CryptoKeyFeature) (CryptoKeysResponse, error) { res, err := c.Request(ctx, http.MethodGet, - "/api/v2/workspaceproxies/me/crypto-keys", + "/api/v2/workspaceproxies/me/crypto-keys", nil, codersdk.WithQueryParam("feature", string(feature)), ) if err != nil { From fa9a75d9d3278faff14c7387dce1ca2285e89199 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 00:21:03 +0000 Subject: [PATCH 14/39] Update cryptokey feature test and migration logic - Enhance testing scenarios for cryptokey features including cases with no keys and specific key states. - Update tests to ensure new cryptokey features are handled correctly. - Remove outdated migration scripts for cryptokey features as they are not required anymore. - Refactor workspace proxy keys to only allow whitelisted cryptokey features, improving security and stability. --- coderd/cryptokeys/rotate_internal_test.go | 17 ++-- coderd/database/dbgen/dbgen.go | 2 + .../000264_cryptokey_features.down.sql | 18 ----- .../000264_cryptokey_features.up.sql | 18 ----- enterprise/coderd/workspaceproxy.go | 9 ++- enterprise/coderd/workspaceproxy_test.go | 79 +++++++++++++++---- 6 files changed, 84 insertions(+), 59 deletions(-) delete mode 100644 coderd/database/migrations/000264_cryptokey_features.down.sql delete mode 100644 coderd/database/migrations/000264_cryptokey_features.up.sql diff --git a/coderd/cryptokeys/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go index ced266477746a..e427a3c6216ac 100644 --- a/coderd/cryptokeys/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -365,9 +365,11 @@ func Test_rotateKeys(t *testing.T) { now := dbnow(clock) - // We'll test a scenario where one feature has no valid keys. - // Another has a key that should be rotate. And one that - // has a valid key that shouldn't trigger an action. + // We'll test a scenario where: + // - One feature has no valid keys. + // - One has a key that should be rotated. + // - One has a valid key that shouldn't trigger an action. + // - One has no keys at all. _ = dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureTailnetResume, StartsAt: now.Add(-keyDuration), @@ -377,6 +379,7 @@ func Test_rotateKeys(t *testing.T) { Valid: false, }, }) + // Generate another deleted key to ensure we insert after the latest sequence. deletedKey := dbgen.CryptoKey(t, db, database.CryptoKey{ Feature: database.CryptoKeyFeatureTailnetResume, StartsAt: now.Add(-keyDuration), @@ -406,7 +409,7 @@ func Test_rotateKeys(t *testing.T) { keys, err := db.GetCryptoKeys(ctx) require.NoError(t, err) - require.Len(t, keys, 4) + require.Len(t, keys, 5) kbf, err := keysByFeature(keys, database.AllCryptoKeyFeatureValues()) require.NoError(t, err) @@ -418,12 +421,14 @@ func Test_rotateKeys(t *testing.T) { // No existing key for tailnet resume should've // caused a key to be inserted. require.Len(t, kbf[database.CryptoKeyFeatureTailnetResume], 1) + require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceAppsToken], 1) oidcKey := kbf[database.CryptoKeyFeatureOIDCConvert][0] tailnetKey := kbf[database.CryptoKeyFeatureTailnetResume][0] + appTokenKey := kbf[database.CryptoKeyFeatureWorkspaceAppsToken][0] requireKey(t, oidcKey, database.CryptoKeyFeatureOIDCConvert, now, nullTime, validKey.Sequence) requireKey(t, tailnetKey, database.CryptoKeyFeatureTailnetResume, now, nullTime, deletedKey.Sequence+1) - + requireKey(t, appTokenKey, database.CryptoKeyFeatureWorkspaceAppsToken, now, nullTime, 1) newKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][0] oldKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][1] if newKey.Sequence == rotatedKey.Sequence { @@ -589,6 +594,8 @@ func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKey switch key.Feature { case database.CryptoKeyFeatureOIDCConvert: require.Len(t, secret, 64) + case database.CryptoKeyFeatureWorkspaceAppsToken: + require.Len(t, secret, 64) case database.CryptoKeyFeatureWorkspaceAppsAPIKey: require.Len(t, secret, 32) case database.CryptoKeyFeatureTailnetResume: diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 7c28e8fdec371..272cb4f5eb09b 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -1050,6 +1050,8 @@ func newCryptoKeySecret(feature database.CryptoKeyFeature) (string, error) { switch feature { case database.CryptoKeyFeatureWorkspaceAppsAPIKey: return generateCryptoKey(32) + case database.CryptoKeyFeatureWorkspaceAppsToken: + return generateCryptoKey(64) case database.CryptoKeyFeatureOIDCConvert: return generateCryptoKey(64) case database.CryptoKeyFeatureTailnetResume: diff --git a/coderd/database/migrations/000264_cryptokey_features.down.sql b/coderd/database/migrations/000264_cryptokey_features.down.sql deleted file mode 100644 index 7cdd00d222da8..0000000000000 --- a/coderd/database/migrations/000264_cryptokey_features.down.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Step 1: Remove the new entries from crypto_keys table -DELETE FROM crypto_keys -WHERE feature IN ('workspace_apps_token', 'workspace_apps_api_key'); - -CREATE TYPE old_crypto_key_feature AS ENUM ( - 'workspace_apps', - 'oidc_convert', - 'tailnet_resume' -); - -ALTER TABLE crypto_keys - ALTER COLUMN feature TYPE old_crypto_key_feature - USING (feature::text::old_crypto_key_feature); - -DROP TYPE crypto_key_feature; - -ALTER TYPE old_crypto_key_feature RENAME TO crypto_key_feature; - diff --git a/coderd/database/migrations/000264_cryptokey_features.up.sql b/coderd/database/migrations/000264_cryptokey_features.up.sql deleted file mode 100644 index bca75d220d0c7..0000000000000 --- a/coderd/database/migrations/000264_cryptokey_features.up.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Create a new enum type with the desired values -CREATE TYPE new_crypto_key_feature AS ENUM ( - 'workspace_apps_token', - 'workspace_apps_api_key', - 'oidc_convert', - 'tailnet_resume' -); - -DELETE FROM crypto_keys WHERE feature = 'workspace_apps'; - --- Drop the old type and rename the new one -ALTER TABLE crypto_keys - ALTER COLUMN feature TYPE new_crypto_key_feature - USING (feature::text::new_crypto_key_feature); - -DROP TYPE crypto_key_feature; - -ALTER TYPE new_crypto_key_feature RENAME TO crypto_key_feature; diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go index b7411d9922220..4008de69e4faa 100644 --- a/enterprise/coderd/workspaceproxy.go +++ b/enterprise/coderd/workspaceproxy.go @@ -34,6 +34,13 @@ import ( "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" ) +// whitelistedCryptoKeyFeatures is a list of crypto key features that are +// allowed to be queried with workspace proxies. +var whitelistedCryptoKeyFeatures = []database.CryptoKeyFeature{ + database.CryptoKeyFeatureWorkspaceAppsToken, + database.CryptoKeyFeatureWorkspaceAppsAPIKey, +} + // forceWorkspaceProxyHealthUpdate forces an update of the proxy health. // This is useful when a proxy is created or deleted. Errors will be logged. func (api *API) forceWorkspaceProxyHealthUpdate(ctx context.Context) { @@ -736,7 +743,7 @@ func (api *API) workspaceProxyCryptoKeys(rw http.ResponseWriter, r *http.Request return } - if !slices.Contains(database.AllCryptoKeyFeatureValues(), feature) { + if !slices.Contains(whitelistedCryptoKeyFeatures, feature) { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ Message: fmt.Sprintf("Invalid feature: %q", feature), }) diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index ad6bf017ea2af..94c8f82726374 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -918,14 +918,14 @@ func TestGetCryptoKeys(t *testing.T) { StartsAt: now.Add(-time.Hour), Sequence: 2, }) - key1 := db2sdk.CryptoKey(expectedKey1) + encryptionKey := db2sdk.CryptoKey(expectedKey1) expectedKey2 := dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, StartsAt: now, Sequence: 3, }) - key2 := db2sdk.CryptoKey(expectedKey2) + signingKey := db2sdk.CryptoKey(expectedKey2) // Create a deleted key. _ = dbgen.CryptoKey(t, db, database.CryptoKey{ @@ -935,19 +935,7 @@ func TestGetCryptoKeys(t *testing.T) { String: "secret1", Valid: false, }, - Sequence: 1, - }) - - // Create a key with different features. - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureTailnetResume, - StartsAt: now.Add(-time.Hour), - Sequence: 1, - }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureOIDCConvert, - StartsAt: now.Add(-time.Hour), - Sequence: 1, + Sequence: 4, }) proxy := coderdenttest.NewWorkspaceProxyReplica(t, api, cclient, &coderdenttest.ProxyOptions{ @@ -957,8 +945,53 @@ func TestGetCryptoKeys(t *testing.T) { keys, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) require.NotEmpty(t, keys) + // 1 key is generated on startup, the other is the one we generated for our test. require.Equal(t, 2, len(keys.CryptoKeys)) - requireContainsKeys(t, keys.CryptoKeys, key1, key2) + requireContainsKeys(t, keys.CryptoKeys, encryptionKey) + requireNotContainsKeys(t, keys.CryptoKeys, signingKey) + + keys, err = proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsToken) + require.NoError(t, err) + require.NotEmpty(t, keys) + requireContainsKeys(t, keys.CryptoKeys, signingKey) + requireNotContainsKeys(t, keys.CryptoKeys, encryptionKey) + }) + + t.Run("InvalidFeature", func(t *testing.T) { + t.Parallel() + + ctx := testutil.Context(t, testutil.WaitMedium) + db, pubsub := dbtestutil.NewDB(t) + cclient, _, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Database: db, + Pubsub: pubsub, + IncludeProvisionerDaemon: true, + }, + LicenseOptions: &coderdenttest.LicenseOptions{ + Features: license.Features{ + codersdk.FeatureWorkspaceProxy: 1, + }, + }, + }) + + proxy := coderdenttest.NewWorkspaceProxyReplica(t, api, cclient, &coderdenttest.ProxyOptions{ + Name: testutil.GetRandomName(t), + }) + + _, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureOIDCConvert) + require.Error(t, err) + var sdkErr *codersdk.Error + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) + _, err = proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureTailnetResume) + require.Error(t, err) + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) + _, err = proxy.SDKClient.CryptoKeys(ctx, "invalid") + require.Error(t, err) + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, http.StatusBadRequest, sdkErr.StatusCode()) }) t.Run("Unauthorized", func(t *testing.T) { @@ -994,6 +1027,18 @@ func TestGetCryptoKeys(t *testing.T) { }) } +func requireNotContainsKeys(t *testing.T, keys []codersdk.CryptoKey, unexpected ...codersdk.CryptoKey) { + t.Helper() + + for _, expectedKey := range unexpected { + for _, key := range keys { + if key.Feature == expectedKey.Feature && key.Sequence == expectedKey.Sequence { + t.Fatalf("unexpected key %+v found", expectedKey) + } + } + } +} + func requireContainsKeys(t *testing.T, keys []codersdk.CryptoKey, expected ...codersdk.CryptoKey) { t.Helper() From 94987b69c67faefc9bb6784ea8a216fa480b32c3 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 00:23:54 +0000 Subject: [PATCH 15/39] gen --- coderd/apidoc/docs.go | 18 +++++++++++++----- coderd/apidoc/swagger.json | 22 +++++++++++++++++----- docs/reference/api/schemas.md | 19 +++++++++---------- site/src/api/typesGenerated.ts | 4 ++-- 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 90bb94e8d132a..07b194fc1bb9f 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -7646,6 +7646,15 @@ const docTemplate = `{ ], "summary": "Get workspace proxy crypto keys", "operationId": "get-workspace-proxy-crypto-keys", + "parameters": [ + { + "type": "string", + "description": "Feature key", + "name": "feature", + "in": "query", + "required": true + } + ], "responses": { "200": { "description": "OK", @@ -10009,12 +10018,14 @@ const docTemplate = `{ "codersdk.CryptoKeyFeature": { "type": "string", "enum": [ - "workspace_apps", + "workspace_apps_api_key", + "workspace_apps_token", "oidc_convert", "tailnet_resume" ], "x-enum-varnames": [ - "CryptoKeyFeatureWorkspaceApp", + "CryptoKeyFeatureWorkspaceAppsAPIKey", + "CryptoKeyFeatureWorkspaceAppsToken", "CryptoKeyFeatureOIDCConvert", "CryptoKeyFeatureTailnetResume" ] @@ -16242,9 +16253,6 @@ const docTemplate = `{ "wsproxysdk.RegisterWorkspaceProxyResponse": { "type": "object", "properties": { - "app_security_key": { - "type": "string" - }, "derp_force_websockets": { "type": "boolean" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 7429cef850c0a..77b68fbbe85db 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -6758,6 +6758,15 @@ "tags": ["Enterprise"], "summary": "Get workspace proxy crypto keys", "operationId": "get-workspace-proxy-crypto-keys", + "parameters": [ + { + "type": "string", + "description": "Feature key", + "name": "feature", + "in": "query", + "required": true + } + ], "responses": { "200": { "description": "OK", @@ -8912,9 +8921,15 @@ }, "codersdk.CryptoKeyFeature": { "type": "string", - "enum": ["workspace_apps", "oidc_convert", "tailnet_resume"], + "enum": [ + "workspace_apps_api_key", + "workspace_apps_token", + "oidc_convert", + "tailnet_resume" + ], "x-enum-varnames": [ - "CryptoKeyFeatureWorkspaceApp", + "CryptoKeyFeatureWorkspaceAppsAPIKey", + "CryptoKeyFeatureWorkspaceAppsToken", "CryptoKeyFeatureOIDCConvert", "CryptoKeyFeatureTailnetResume" ] @@ -14851,9 +14866,6 @@ "wsproxysdk.RegisterWorkspaceProxyResponse": { "type": "object", "properties": { - "app_security_key": { - "type": "string" - }, "derp_force_websockets": { "type": "boolean" }, diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index ac8636f3e6f46..017d41c90cc4e 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -1453,7 +1453,7 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o ```json { "deletes_at": "2019-08-24T14:15:22Z", - "feature": "workspace_apps", + "feature": "workspace_apps_api_key", "secret": "string", "sequence": 0, "starts_at": "2019-08-24T14:15:22Z" @@ -1473,18 +1473,19 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o ## codersdk.CryptoKeyFeature ```json -"workspace_apps" +"workspace_apps_api_key" ``` ### Properties #### Enumerated Values -| Value | -| ---------------- | -| `workspace_apps` | -| `oidc_convert` | -| `tailnet_resume` | +| Value | +| ------------------------ | +| `workspace_apps_api_key` | +| `workspace_apps_token` | +| `oidc_convert` | +| `tailnet_resume` | ## codersdk.CustomRoleRequest @@ -9892,7 +9893,7 @@ _None_ "crypto_keys": [ { "deletes_at": "2019-08-24T14:15:22Z", - "feature": "workspace_apps", + "feature": "workspace_apps_api_key", "secret": "string", "sequence": 0, "starts_at": "2019-08-24T14:15:22Z" @@ -9970,7 +9971,6 @@ _None_ ```json { - "app_security_key": "string", "derp_force_websockets": true, "derp_map": { "homeParams": { @@ -10051,7 +10051,6 @@ _None_ | Name | Type | Required | Restrictions | Description | | ----------------------- | --------------------------------------------- | -------- | ------------ | -------------------------------------------------------------------------------------- | -| `app_security_key` | string | false | | | | `derp_force_websockets` | boolean | false | | | | `derp_map` | [tailcfg.DERPMap](#tailcfgderpmap) | false | | | | `derp_mesh_key` | string | false | | | diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 76be331a526cf..cc50bc4838261 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2110,8 +2110,8 @@ export type BuildReason = "autostart" | "autostop" | "initiator" export const BuildReasons: BuildReason[] = ["autostart", "autostop", "initiator"] // From codersdk/deployment.go -export type CryptoKeyFeature = "oidc_convert" | "tailnet_resume" | "workspace_apps" -export const CryptoKeyFeatures: CryptoKeyFeature[] = ["oidc_convert", "tailnet_resume", "workspace_apps"] +export type CryptoKeyFeature = "oidc_convert" | "tailnet_resume" | "workspace_apps_api_key" | "workspace_apps_token" +export const CryptoKeyFeatures: CryptoKeyFeature[] = ["oidc_convert", "tailnet_resume", "workspace_apps_api_key", "workspace_apps_token"] // From codersdk/workspaceagents.go export type DisplayApp = "port_forwarding_helper" | "ssh_helper" | "vscode" | "vscode_insiders" | "web_terminal" From 76561ac268aa050018e7758431308a8338fd125b Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 00:52:47 +0000 Subject: [PATCH 16/39] Refactor cryptokeys cache to include key reader context Enhances the cache functionality by wrapping the context with a key reader, ensuring proper authorization checks during cryptographic operations. This change aligns cache behavior with security practices. --- coderd/cryptokeys/cache.go | 8 ++ coderd/workspaceapps/token_test.go | 136 ----------------------------- 2 files changed, 8 insertions(+), 136 deletions(-) diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 601916fec9419..1f6d3d3acc16f 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -155,6 +155,8 @@ func (c *cache) EncryptingKey(ctx context.Context) (string, interface{}, error) return "", nil, ErrInvalidFeature } + //nolint:gocritic // cache can only rotate crypto keys. + ctx = dbauthz.AsKeyReader(ctx) return c.cryptoKey(ctx, latestSequence) } @@ -168,6 +170,8 @@ func (c *cache) DecryptingKey(ctx context.Context, id string) (interface{}, erro return nil, xerrors.Errorf("parse id: %w", err) } + //nolint:gocritic // cache can only rotate crypto keys. + ctx = dbauthz.AsKeyReader(ctx) _, secret, err := c.cryptoKey(ctx, int32(seq)) if err != nil { return nil, xerrors.Errorf("crypto key: %w", err) @@ -180,6 +184,8 @@ func (c *cache) SigningKey(ctx context.Context) (string, interface{}, error) { return "", nil, ErrInvalidFeature } + //nolint:gocritic // cache can only rotate crypto keys. + ctx = dbauthz.AsKeyReader(ctx) return c.cryptoKey(ctx, latestSequence) } @@ -198,6 +204,8 @@ func (c *cache) VerifyingKey(ctx context.Context, id string) (interface{}, error return nil, xerrors.Errorf("crypto key: %w", err) } + //nolint:gocritic // cache can only rotate crypto keys. + ctx = dbauthz.AsKeyReader(ctx) return secret, nil } diff --git a/coderd/workspaceapps/token_test.go b/coderd/workspaceapps/token_test.go index aa51366d661dd..46a1b9607f5ae 100644 --- a/coderd/workspaceapps/token_test.go +++ b/coderd/workspaceapps/token_test.go @@ -283,142 +283,6 @@ func Test_TokenMatchesRequest(t *testing.T) { } } -func Test_GenerateToken(t *testing.T) { - t.Parallel() - - t.Run("SetExpiry", func(t *testing.T) { - t.Parallel() - - ctx := testutil.Context(t, testutil.WaitShort) - signer := newSigner(t) - - tokenStr, err := jwtutils.Sign(ctx, signer, workspaceapps.SignedToken{ - Request: workspaceapps.Request{ - AccessMethod: workspaceapps.AccessMethodPath, - BasePath: "/app", - UsernameOrID: "foo", - WorkspaceNameOrID: "bar", - AgentNameOrID: "baz", - AppSlugOrPort: "qux", - }, - - UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), - WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), - AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), - AppURL: "http://127.0.0.1:8080", - }) - require.NoError(t, err) - - var token workspaceapps.SignedToken - err = jwtutils.Verify(ctx, signer, tokenStr, &token) - require.NoError(t, err) - - require.WithinDuration(t, time.Now().Add(time.Minute), token.Expiry.Time(), 15*time.Second) - }) - - future := time.Now().Add(time.Hour) - cases := []struct { - name string - token workspaceapps.SignedToken - parseErrContains string - }{ - { - name: "OK1", - token: workspaceapps.SignedToken{ - Claims: jwt.Claims{ - Expiry: jwt.NewNumericDate(future), - }, - Request: workspaceapps.Request{ - AccessMethod: workspaceapps.AccessMethodPath, - BasePath: "/app", - UsernameOrID: "foo", - WorkspaceNameOrID: "bar", - AgentNameOrID: "baz", - AppSlugOrPort: "qux", - }, - - UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), - WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), - AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), - AppURL: "http://127.0.0.1:8080", - }, - }, - { - name: "OK2", - token: workspaceapps.SignedToken{ - Claims: jwt.Claims{ - Expiry: jwt.NewNumericDate(future), - }, - Request: workspaceapps.Request{ - AccessMethod: workspaceapps.AccessMethodSubdomain, - BasePath: "/", - UsernameOrID: "oof", - WorkspaceNameOrID: "rab", - AgentNameOrID: "zab", - AppSlugOrPort: "xuq", - }, - - UserID: uuid.MustParse("6fa684a3-11aa-49fd-8512-ab527bd9b900"), - WorkspaceID: uuid.MustParse("b2d816cc-505c-441d-afdf-dae01781bc0b"), - AgentID: uuid.MustParse("6c4396e1-af88-4a8a-91a3-13ea54fc29fb"), - AppURL: "http://localhost:9090", - }, - }, - { - name: "Expired", - token: workspaceapps.SignedToken{ - Claims: jwt.Claims{ - Expiry: jwt.NewNumericDate(time.Now().Add(-time.Hour)), - }, - - Request: workspaceapps.Request{ - AccessMethod: workspaceapps.AccessMethodSubdomain, - BasePath: "/", - UsernameOrID: "foo", - WorkspaceNameOrID: "bar", - AgentNameOrID: "baz", - AppSlugOrPort: "qux", - }, - - UserID: uuid.MustParse("b1530ba9-76f3-415e-b597-4ddd7cd466a4"), - WorkspaceID: uuid.MustParse("1e6802d3-963e-45ac-9d8c-bf997016ffed"), - AgentID: uuid.MustParse("9ec18681-d2c9-4c9e-9186-f136efb4edbe"), - AppURL: "http://127.0.0.1:8080", - }, - parseErrContains: "token expired", - }, - } - - signer := newSigner(t) - for _, c := range cases { - c := c - - t.Run(c.name, func(t *testing.T) { - t.Parallel() - - ctx := testutil.Context(t, testutil.WaitShort) - str, err := jwtutils.Sign(ctx, signer, c.token) - require.NoError(t, err) - - // Tokens aren't deterministic as they have a random nonce, so we - // can't compare them directly. - - var token workspaceapps.SignedToken - err = jwtutils.Verify(ctx, signer, str, &token) - if c.parseErrContains != "" { - require.Error(t, err) - require.ErrorContains(t, err, c.parseErrContains) - } else { - require.NoError(t, err) - // normalize the expiry - require.WithinDuration(t, c.token.Expiry.Time(), token.Expiry.Time(), 10*time.Second) - c.token.Expiry = token.Expiry - require.Equal(t, c.token, token) - } - }) - } -} - func Test_FromRequest(t *testing.T) { t.Parallel() From 53dcf36a94a867ea55186492389cd6070f33ff92 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 00:54:05 +0000 Subject: [PATCH 17/39] Refactor jwtutils to remove redundant key reader --- coderd/userauth.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coderd/userauth.go b/coderd/userauth.go index f4e74641c71c4..5d312c3cd9cb4 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -169,7 +169,7 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { //nolint:gocritic // We need to read the system signing key // in order to sign the token. - token, err := jwtutils.Sign(dbauthz.AsKeyReader(ctx), api.OIDCConvertKeyCache, claims) + token, err := jwtutils.Sign(ctx, api.OIDCConvertKeyCache, claims) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Internal error signing state jwt.", @@ -1681,7 +1681,7 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data //nolint:gocritic // We need to read the system signing key // in order to verify the token. - err = jwtutils.Verify(dbauthz.AsKeyReader(ctx), api.OIDCConvertKeyCache, jwtCookie.Value, &claims) + err = jwtutils.Verify(ctx, api.OIDCConvertKeyCache, jwtCookie.Value, &claims) if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) { // These errors are probably because the user is mixing 2 coder deployments. return database.User{}, idpsync.HTTPError{ From c656d001a7ba94dd13138370a0d13f928db1ede4 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 01:00:44 +0000 Subject: [PATCH 18/39] Remove unused cryptokey feature fixtures --- .../fixtures/000264_cryptokey_features.up.sql | 40 ------------------- 1 file changed, 40 deletions(-) delete mode 100644 coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql deleted file mode 100644 index 5cb2cd4c95509..0000000000000 --- a/coderd/database/migrations/testdata/fixtures/000264_cryptokey_features.up.sql +++ /dev/null @@ -1,40 +0,0 @@ -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -VALUES ( - 'workspace_apps_token', - 1, - 'abc', - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - '2100-01-01 00:00:00 UTC'::timestamptz -); - -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -VALUES ( - 'workspace_apps_api_key', - 1, - 'def', - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - '2100-01-01 00:00:00 UTC'::timestamptz -); - -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -VALUES ( - 'oidc_convert', - 2, - 'ghi', - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - '2100-01-01 00:00:00 UTC'::timestamptz -); - -INSERT INTO crypto_keys (feature, sequence, secret, secret_key_id, starts_at, deletes_at) -VALUES ( - 'tailnet_resume', - 2, - 'jkl', - NULL, - '1970-01-01 00:00:00 UTC'::timestamptz, - '2100-01-01 00:00:00 UTC'::timestamptz -); - From e7cfb468e38e79916f4d51690b2443ea83cb77f2 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 01:09:33 +0000 Subject: [PATCH 19/39] Refactor cryptokeys comments and variable typo Adjust comments to accurately describe cache functionality and fix a typo in the `New` function parameters for clarity. --- coderd/coderd.go | 6 +++--- coderd/cryptokeys/cache.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index b90b40221b560..f17b4ef2d24f0 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -683,9 +683,9 @@ func New(options *Options) *API { AgentProvider: api.agentProvider, StatsCollector: workspaceapps.NewStatsCollector(options.WorkspaceAppsStatsCollectorOptions), - DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), - SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), - APIKeyEncryptionKey: options.AppEncryptionKeyCache, + DisablePathApps: options.DeploymentValues.DisablePathApps.Value(), + SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(), + APIKeyEncryptionKeycache: options.AppEncryptionKeyCache, } apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{ diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 1f6d3d3acc16f..083bf6d4c6c5d 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -155,7 +155,7 @@ func (c *cache) EncryptingKey(ctx context.Context) (string, interface{}, error) return "", nil, ErrInvalidFeature } - //nolint:gocritic // cache can only rotate crypto keys. + //nolint:gocritic // cache can only read crypto keys. ctx = dbauthz.AsKeyReader(ctx) return c.cryptoKey(ctx, latestSequence) } @@ -170,7 +170,7 @@ func (c *cache) DecryptingKey(ctx context.Context, id string) (interface{}, erro return nil, xerrors.Errorf("parse id: %w", err) } - //nolint:gocritic // cache can only rotate crypto keys. + //nolint:gocritic // cache can only read crypto keys. ctx = dbauthz.AsKeyReader(ctx) _, secret, err := c.cryptoKey(ctx, int32(seq)) if err != nil { @@ -184,7 +184,7 @@ func (c *cache) SigningKey(ctx context.Context) (string, interface{}, error) { return "", nil, ErrInvalidFeature } - //nolint:gocritic // cache can only rotate crypto keys. + //nolint:gocritic // cache can only read crypto keys. ctx = dbauthz.AsKeyReader(ctx) return c.cryptoKey(ctx, latestSequence) } @@ -204,7 +204,7 @@ func (c *cache) VerifyingKey(ctx context.Context, id string) (interface{}, error return nil, xerrors.Errorf("crypto key: %w", err) } - //nolint:gocritic // cache can only rotate crypto keys. + //nolint:gocritic // cache can only read crypto keys. ctx = dbauthz.AsKeyReader(ctx) return secret, nil } From 6432b0d726309e8492b822a3ff0fda71a7168a15 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 01:23:56 +0000 Subject: [PATCH 20/39] fix comments --- coderd/cryptokeys/cache.go | 5 ++--- coderd/userauth.go | 4 ---- enterprise/wsproxy/wsproxy_test.go | 6 ------ 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 083bf6d4c6c5d..03414841cc8e6 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -198,14 +198,13 @@ func (c *cache) VerifyingKey(ctx context.Context, id string) (interface{}, error if err != nil { return nil, xerrors.Errorf("parse id: %w", err) } - + //nolint:gocritic // cache can only read crypto keys. + ctx = dbauthz.AsKeyReader(ctx) _, secret, err := c.cryptoKey(ctx, int32(seq)) if err != nil { return nil, xerrors.Errorf("crypto key: %w", err) } - //nolint:gocritic // cache can only read crypto keys. - ctx = dbauthz.AsKeyReader(ctx) return secret, nil } diff --git a/coderd/userauth.go b/coderd/userauth.go index 5d312c3cd9cb4..eaf2aa3659ae8 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -167,8 +167,6 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { ToLoginType: req.ToType, } - //nolint:gocritic // We need to read the system signing key - // in order to sign the token. token, err := jwtutils.Sign(ctx, api.OIDCConvertKeyCache, claims) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ @@ -1679,8 +1677,6 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data } var claims OAuthConvertStateClaims - //nolint:gocritic // We need to read the system signing key - // in order to verify the token. err = jwtutils.Verify(ctx, api.OIDCConvertKeyCache, jwtCookie.Value, &claims) if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) { // These errors are probably because the user is mixing 2 coder deployments. diff --git a/enterprise/wsproxy/wsproxy_test.go b/enterprise/wsproxy/wsproxy_test.go index 64aca14377318..3d3926c5afae7 100644 --- a/enterprise/wsproxy/wsproxy_test.go +++ b/enterprise/wsproxy/wsproxy_test.go @@ -8,7 +8,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "runtime/debug" "testing" "time" @@ -322,11 +321,6 @@ resourceLoop: func TestDERPEndToEnd(t *testing.T) { t.Parallel() - defer func() { - if r := recover(); r != nil { - debug.PrintStack() - } - }() deploymentValues := coderdtest.DeploymentValues(t) deploymentValues.Experiments = []string{ From 9ad187d8754cbec10bff9fc216e85f121a2164d3 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 01:49:02 +0000 Subject: [PATCH 21/39] move rotator out of coderd --- cli/server.go | 6 ++++++ coderd/coderd.go | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/server.go b/cli/server.go index 3c6c14e300f65..ebbb9806a92c3 100644 --- a/cli/server.go +++ b/cli/server.go @@ -746,6 +746,12 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("set deployment id: %w", err) } + // Start a background process that rotates keys. + err = cryptokeys.StartRotator(ctx, options.Logger.Named("keyrotator"), options.Database) + if err != nil { + return xerrors.Errorf("start key rotator: %w", err) + } + fetcher := &cryptokeys.DBFetcher{ DB: options.Database, } diff --git a/coderd/coderd.go b/coderd/coderd.go index f17b4ef2d24f0..351c57aa21175 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -449,12 +449,6 @@ func New(options *Options) *API { panic(xerrors.Errorf("get deployment ID: %w", err)) } - // Start a background process that rotates keys. - err = cryptokeys.StartRotator(ctx, options.Logger.Named("keyrotator"), options.Database) - if err != nil { - options.Logger.Fatal(ctx, "start key rotator", slog.Error(err)) - } - fetcher := &cryptokeys.DBFetcher{ DB: options.Database, } From d16e98f8f75222d0244e0249dab5bd53949e5c12 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 01:50:44 +0000 Subject: [PATCH 22/39] remove composite jwtutil interfaces --- coderd/cryptokeys/cache.go | 54 ++++++++++++++++++- coderd/jwtutils/jwe.go | 5 -- coderd/jwtutils/jws.go | 21 -------- coderd/workspaceagents_test.go | 4 +- coderd/workspaceapps/db.go | 5 +- coderd/workspaceapps/proxy.go | 7 +-- coderd/workspaceapps/token.go | 3 +- coderd/workspaceapps/token_test.go | 5 +- .../workspacesdk/connector_internal_test.go | 6 +-- enterprise/wsproxy/tokenprovider.go | 13 ++--- enterprise/wsproxy/wsproxy.go | 20 +++---- tailnet/resume.go | 7 +-- tailnet/resume_test.go | 6 +-- 13 files changed, 93 insertions(+), 63 deletions(-) diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 03414841cc8e6..497364cdb1c7f 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -13,7 +13,6 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/codersdk" "github.com/coder/quartz" @@ -73,7 +72,7 @@ func (d *DBFetcher) Fetch(ctx context.Context, feature codersdk.CryptoKeyFeature return nil, xerrors.Errorf("get crypto keys by feature: %w", err) } - return db2sdk.CryptoKeys(keys), nil + return toSDKKeys(keys), nil } // cache implements the caching functionality for both signing and encryption keys. @@ -378,3 +377,54 @@ func (c *cache) Close() error { return nil } + +// StaticKey fulfills the SigningKeycache and EncryptionKeycache interfaces. Useful for testing. +type StaticKey struct { + ID string + Key interface{} +} + +func (s StaticKey) SigningKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKey) VerifyingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} + +func (s StaticKey) EncryptingKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKey) DecryptingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} + +func (s StaticKey) Close() error { + return nil +} + +// We have to do this to avoid a circular dependency on db2sdk (cryptokeys -> db2sdk -> tailnet -> cryptokeys) +func toSDKKeys(keys []database.CryptoKey) []codersdk.CryptoKey { + into := make([]codersdk.CryptoKey, 0, len(keys)) + for _, key := range keys { + into = append(into, toSDK(key)) + } + return into +} + +func toSDK(key database.CryptoKey) codersdk.CryptoKey { + return codersdk.CryptoKey{ + Feature: codersdk.CryptoKeyFeature(key.Feature), + Sequence: key.Sequence, + StartsAt: key.StartsAt, + DeletesAt: key.DeletesAt.Time, + Secret: key.Secret.String, + } +} diff --git a/coderd/jwtutils/jwe.go b/coderd/jwtutils/jwe.go index 5ac62946f52b7..704dc3d4011d6 100644 --- a/coderd/jwtutils/jwe.go +++ b/coderd/jwtutils/jwe.go @@ -15,11 +15,6 @@ const ( encryptContentAlgo = jose.A256GCM ) -type EncryptingKeyManager interface { - EncryptKeyProvider - DecryptKeyProvider -} - type EncryptKeyProvider interface { EncryptingKey(ctx context.Context) (id string, key interface{}, err error) } diff --git a/coderd/jwtutils/jws.go b/coderd/jwtutils/jws.go index 6ea29e74a6956..553d0da0aeb2d 100644 --- a/coderd/jwtutils/jws.go +++ b/coderd/jwtutils/jws.go @@ -24,27 +24,6 @@ const ( signingAlgo = jose.HS512 ) -type StaticKeyManager struct { - ID string - Key interface{} -} - -func (s StaticKeyManager) SigningKey(_ context.Context) (string, interface{}, error) { - return s.ID, s.Key, nil -} - -func (s StaticKeyManager) VerifyingKey(_ context.Context, id string) (interface{}, error) { - if id != s.ID { - return nil, xerrors.Errorf("invalid id %q", id) - } - return s.Key, nil -} - -type SigningKeyManager interface { - SigningKeyProvider - VerifyKeyProvider -} - type SigningKeyProvider interface { SigningKey(ctx context.Context) (id string, key interface{}, err error) } diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 225ec608f783a..04aa5191901ec 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -28,6 +28,7 @@ import ( agentproto "github.com/coder/coder/v2/agent/proto" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/coderdtest/oidctest" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbfake" @@ -36,7 +37,6 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/database/pubsub" "github.com/coder/coder/v2/coderd/externalauth" - "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/coder/v2/codersdk/workspacesdk" @@ -558,7 +558,7 @@ func TestWorkspaceAgentClientCoordinate_ResumeToken(t *testing.T) { logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() - mgr := jwtutils.StaticKeyManager{ + mgr := cryptokeys.StaticKey{ ID: uuid.New().String(), Key: resumeTokenSigningKey[:], } diff --git a/coderd/workspaceapps/db.go b/coderd/workspaceapps/db.go index 2018eba847651..b3f223dec9ebd 100644 --- a/coderd/workspaceapps/db.go +++ b/coderd/workspaceapps/db.go @@ -16,6 +16,7 @@ import ( "github.com/go-jose/go-jose/v4/jwt" "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/httpapi" @@ -38,12 +39,12 @@ type DBTokenProvider struct { DeploymentValues *codersdk.DeploymentValues OAuth2Configs *httpmw.OAuth2Configs WorkspaceAgentInactiveTimeout time.Duration - TokenSigner jwtutils.SigningKeyManager + TokenSigner cryptokeys.SigningKeycache } var _ SignedTokenProvider = &DBTokenProvider{} -func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authorizer, db database.Store, cfg *codersdk.DeploymentValues, oauth2Cfgs *httpmw.OAuth2Configs, workspaceAgentInactiveTimeout time.Duration, signer jwtutils.SigningKeyManager) SignedTokenProvider { +func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authorizer, db database.Store, cfg *codersdk.DeploymentValues, oauth2Cfgs *httpmw.OAuth2Configs, workspaceAgentInactiveTimeout time.Duration, signer cryptokeys.SigningKeycache) SignedTokenProvider { if workspaceAgentInactiveTimeout == 0 { workspaceAgentInactiveTimeout = 1 * time.Minute } diff --git a/coderd/workspaceapps/proxy.go b/coderd/workspaceapps/proxy.go index ad6d445e15310..84cea4fa86678 100644 --- a/coderd/workspaceapps/proxy.go +++ b/coderd/workspaceapps/proxy.go @@ -21,6 +21,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/agent/agentssh" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" @@ -100,8 +101,8 @@ type Server struct { HostnameRegex *regexp.Regexp RealIPConfig *httpmw.RealIPConfig - SignedTokenProvider SignedTokenProvider - APIKeyEncryptionKey jwtutils.EncryptingKeyManager + SignedTokenProvider SignedTokenProvider + APIKeyEncryptionKeycache cryptokeys.EncryptionKeycache // DisablePathApps disables path-based apps. This is a security feature as path // based apps share the same cookie as the dashboard, and are susceptible to XSS @@ -180,7 +181,7 @@ func (s *Server) handleAPIKeySmuggling(rw http.ResponseWriter, r *http.Request, // Exchange the encoded API key for a real one. var payload EncryptedAPIKeyPayload - err := jwtutils.Decrypt(ctx, s.APIKeyEncryptionKey, encryptedAPIKey, &payload, jwtutils.WithDecryptExpected(jwt.Expected{ + err := jwtutils.Decrypt(ctx, s.APIKeyEncryptionKeycache, encryptedAPIKey, &payload, jwtutils.WithDecryptExpected(jwt.Expected{ Time: time.Now(), })) if err != nil { diff --git a/coderd/workspaceapps/token.go b/coderd/workspaceapps/token.go index ee8101590f88e..08e2df22b7269 100644 --- a/coderd/workspaceapps/token.go +++ b/coderd/workspaceapps/token.go @@ -8,6 +8,7 @@ import ( "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" ) @@ -54,7 +55,7 @@ type EncryptedAPIKeyPayload struct { // FromRequest returns the signed token from the request, if it exists and is // valid. The caller must check that the token matches the request. -func FromRequest(r *http.Request, mgr jwtutils.SigningKeyManager) (*SignedToken, bool) { +func FromRequest(r *http.Request, mgr cryptokeys.SigningKeycache) (*SignedToken, bool) { // Get all signed app tokens from the request. This includes the query // parameter and all matching cookies sent with the request. If there are // somehow multiple signed app token cookies, we want to try all of them diff --git a/coderd/workspaceapps/token_test.go b/coderd/workspaceapps/token_test.go index 46a1b9607f5ae..5bd7abeb29637 100644 --- a/coderd/workspaceapps/token_test.go +++ b/coderd/workspaceapps/token_test.go @@ -15,6 +15,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" ) @@ -343,10 +344,10 @@ func Test_FromRequest(t *testing.T) { }) } -func newSigner(t *testing.T) jwtutils.SigningKeyManager { +func newSigner(t *testing.T) cryptokeys.StaticKey { t.Helper() - return jwtutils.StaticKeyManager{ + return cryptokeys.StaticKey{ ID: "test", Key: generateSecret(t, 64), } diff --git a/codersdk/workspacesdk/connector_internal_test.go b/codersdk/workspacesdk/connector_internal_test.go index db48f5ef7fc1b..2d70278095653 100644 --- a/codersdk/workspacesdk/connector_internal_test.go +++ b/codersdk/workspacesdk/connector_internal_test.go @@ -24,8 +24,8 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/apiversion" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/httpapi" - "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/tailnet/proto" @@ -166,7 +166,7 @@ func TestTailnetAPIConnector_ResumeToken(t *testing.T) { clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - mgr := jwtutils.StaticKeyManager{ + mgr := cryptokeys.StaticKey{ ID: uuid.New().String(), Key: resumeTokenSigningKey[:], } @@ -285,7 +285,7 @@ func TestTailnetAPIConnector_ResumeTokenFailure(t *testing.T) { clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - mgr := jwtutils.StaticKeyManager{ + mgr := cryptokeys.StaticKey{ ID: uuid.New().String(), Key: resumeTokenSigningKey[:], } diff --git a/enterprise/wsproxy/tokenprovider.go b/enterprise/wsproxy/tokenprovider.go index 8e452f1766aa0..5093c6015725e 100644 --- a/enterprise/wsproxy/tokenprovider.go +++ b/enterprise/wsproxy/tokenprovider.go @@ -7,6 +7,7 @@ import ( "cdr.dev/slog" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk" @@ -19,14 +20,14 @@ type TokenProvider struct { AccessURL *url.URL AppHostname string - Client *wsproxysdk.Client - TokenSigningKey jwtutils.SigningKeyManager - APIKeyEncryptionKey jwtutils.EncryptingKeyManager - Logger slog.Logger + Client *wsproxysdk.Client + TokenSigningKeycache cryptokeys.SigningKeycache + APIKeyEncryptionKeycache cryptokeys.EncryptionKeycache + Logger slog.Logger } func (p *TokenProvider) FromRequest(r *http.Request) (*workspaceapps.SignedToken, bool) { - return workspaceapps.FromRequest(r, p.TokenSigningKey) + return workspaceapps.FromRequest(r, p.TokenSigningKeycache) } func (p *TokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *http.Request, issueReq workspaceapps.IssueTokenRequest) (*workspaceapps.SignedToken, string, bool) { @@ -45,7 +46,7 @@ func (p *TokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *ht // Check that it verifies properly and matches the string. var token workspaceapps.SignedToken - err = jwtutils.Verify(ctx, p.TokenSigningKey, resp.SignedTokenStr, &token) + err = jwtutils.Verify(ctx, p.TokenSigningKeycache, resp.SignedTokenStr, &token) if err != nil { workspaceapps.WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "failed to verify newly generated signed token") return nil, "", false diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go index 5410572802087..4bcd7255ba32b 100644 --- a/enterprise/wsproxy/wsproxy.go +++ b/enterprise/wsproxy/wsproxy.go @@ -303,21 +303,21 @@ func New(ctx context.Context, opts *Options) (*Server, error) { HostnameRegex: opts.AppHostnameRegex, RealIPConfig: opts.RealIPConfig, SignedTokenProvider: &TokenProvider{ - DashboardURL: opts.DashboardURL, - AccessURL: opts.AccessURL, - AppHostname: opts.AppHostname, - Client: client, - TokenSigningKey: signingCache, - APIKeyEncryptionKey: encryptionCache, - Logger: s.Logger.Named("proxy_token_provider"), + DashboardURL: opts.DashboardURL, + AccessURL: opts.AccessURL, + AppHostname: opts.AppHostname, + Client: client, + TokenSigningKeycache: signingCache, + APIKeyEncryptionKeycache: encryptionCache, + Logger: s.Logger.Named("proxy_token_provider"), }, DisablePathApps: opts.DisablePathApps, SecureAuthCookie: opts.SecureAuthCookie, - AgentProvider: agentProvider, - StatsCollector: workspaceapps.NewStatsCollector(opts.StatsCollectorOptions), - APIKeyEncryptionKey: encryptionCache, + AgentProvider: agentProvider, + StatsCollector: workspaceapps.NewStatsCollector(opts.StatsCollectorOptions), + APIKeyEncryptionKeycache: encryptionCache, } derpHandler := derphttp.Handler(derpServer) diff --git a/tailnet/resume.go b/tailnet/resume.go index b79850bfb6a63..6b4ea2417b3da 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -11,6 +11,7 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/tailnet/proto" "github.com/coder/quartz" @@ -28,7 +29,7 @@ func NewInsecureTestResumeTokenProvider() ResumeTokenProvider { if err != nil { panic(err) } - return NewResumeTokenKeyProvider(jwtutils.StaticKeyManager{ + return NewResumeTokenKeyProvider(cryptokeys.StaticKey{ ID: uuid.New().String(), Key: key[:], }, quartz.NewReal(), time.Hour) @@ -51,12 +52,12 @@ func GenerateResumeTokenSigningKey() (ResumeTokenSigningKey, error) { } type ResumeTokenKeyProvider struct { - key jwtutils.SigningKeyManager + key cryptokeys.SigningKeycache clock quartz.Clock expiry time.Duration } -func NewResumeTokenKeyProvider(key jwtutils.SigningKeyManager, clock quartz.Clock, expiry time.Duration) ResumeTokenProvider { +func NewResumeTokenKeyProvider(key cryptokeys.SigningKeycache, clock quartz.Clock, expiry time.Duration) ResumeTokenProvider { if expiry <= 0 { expiry = DefaultResumeTokenExpiry } diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index e896c49dfd8f0..6c25c12eeaf36 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" - "github.com/coder/coder/v2/coderd/jwtutils" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" @@ -91,8 +91,8 @@ func TestResumeTokenKeyProvider(t *testing.T) { }) } -func newKeySigner(key tailnet.ResumeTokenSigningKey) jwtutils.StaticKeyManager { - return jwtutils.StaticKeyManager{ +func newKeySigner(key tailnet.ResumeTokenSigningKey) cryptokeys.StaticKey { + return cryptokeys.StaticKey{ ID: uuid.New().String(), Key: key[:], } From 3809cd5fa2c5777f437f7027505f142fc180b58c Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 17 Oct 2024 02:48:51 +0000 Subject: [PATCH 23/39] fix tests caused by moving rotator initiation out of coderd --- cli/server_test.go | 39 ++++++++++++++++++++++++ coderd/cryptokeys/cache.go | 2 +- coderd/userauth_test.go | 7 ++++- coderd/workspaceapps/db_test.go | 13 ++++++-- coderd/workspaceapps_test.go | 17 +++++++---- enterprise/coderd/workspaceproxy_test.go | 31 +++++++------------ enterprise/workspaceapps_test.go | 15 +++++++++ enterprise/wsproxy/wsproxy_test.go | 26 ++++++++++++++++ scaletest/createworkspaces/run_test.go | 12 ++++++-- scaletest/reconnectingpty/run_test.go | 9 ++++++ scaletest/workspacetraffic/run_test.go | 7 ++++- 11 files changed, 145 insertions(+), 33 deletions(-) diff --git a/cli/server_test.go b/cli/server_test.go index ad6a98038c7bb..6fc083d43bf60 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -44,6 +44,7 @@ import ( "github.com/coder/coder/v2/cli/clitest" "github.com/coder/coder/v2/cli/config" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/telemetry" @@ -1590,6 +1591,44 @@ func TestServer(t *testing.T) { w.RequireSuccess() }) }) + + t.Run("CryptoKeysGenerated", func(t *testing.T) { + t.Parallel() + if testing.Short() { + t.SkipNow() + } + + if runtime.GOOS != "linux" || testing.Short() { + // Skip on non-Linux because it spawns a PostgreSQL instance. + t.SkipNow() + } + connectionURL, closeFunc, err := dbtestutil.Open() + require.NoError(t, err) + defer closeFunc() + + inv, cfg := clitest.New(t, + "server", + "--http-address", ":0", + "--access-url", "http://example.com", + "--postgres-url", connectionURL, + "--cache-dir", t.TempDir(), + ) + + const superDuperLong = testutil.WaitSuperLong * 3 + ctx := testutil.Context(t, superDuperLong) + clitest.Start(t, inv.WithContext(ctx)) + _ = waitAccessURL(t, cfg) + + logger := slogtest.Make(t, nil) + sqldb, err := cli.ConnectToPostgres(ctx, logger, "postgres", connectionURL) + require.NoError(t, err) + defer sqldb.Close() + + db := database.New(sqldb) + keys, err := db.GetCryptoKeys(ctx) + require.NoError(t, err) + require.Len(t, keys, len(database.AllCryptoKeyFeatureValues())) + }) } func TestServer_Production(t *testing.T) { diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 497364cdb1c7f..0f4e88daef357 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -406,7 +406,7 @@ func (s StaticKey) DecryptingKey(_ context.Context, id string) (interface{}, err return s.Key, nil } -func (s StaticKey) Close() error { +func (StaticKey) Close() error { return nil } diff --git a/coderd/userauth_test.go b/coderd/userauth_test.go index 20dfe7f723899..df33ce2fedd96 100644 --- a/coderd/userauth_test.go +++ b/coderd/userauth_test.go @@ -1309,11 +1309,16 @@ func TestUserOIDC(t *testing.T) { cfg.AllowSignups = true }) - client := coderdtest.New(t, &coderdtest.Options{ + client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ Auditor: auditor, OIDCConfig: cfg, }) + // Make sure there's a signing key! + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureOIDCConvert, + }) + owner := coderdtest.CreateFirstUser(t, client) user, userData := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID) diff --git a/coderd/workspaceapps/db_test.go b/coderd/workspaceapps/db_test.go index e648cc38cddff..48709daedcb2a 100644 --- a/coderd/workspaceapps/db_test.go +++ b/coderd/workspaceapps/db_test.go @@ -20,6 +20,9 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" @@ -76,6 +79,7 @@ func Test_ResolveRequest(t *testing.T) { deploymentValues.Dangerous.AllowPathAppSharing = true deploymentValues.Dangerous.AllowPathAppSiteOwnerAccess = true + db, pubsub := dbtestutil.NewDB(t) client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ AppHostname: "*.test.coder.com", DeploymentValues: deploymentValues, @@ -91,13 +95,18 @@ func Test_ResolveRequest(t *testing.T) { "CF-Connecting-IP", }, }, + Database: db, + Pubsub: pubsub, }) t.Cleanup(func() { _ = closer.Close() }) - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium) - t.Cleanup(cancel) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + + ctx := testutil.Context(t, testutil.WaitMedium) firstUser := coderdtest.CreateFirstUser(t, client) me, err := client.User(ctx, codersdk.Me) diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index 1d00b7daa7bd9..a5cec65410303 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -11,7 +11,6 @@ import ( "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbgen" - "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" @@ -181,19 +180,25 @@ func TestWorkspaceApplicationAuth(t *testing.T) { t.Run(c.name, func(t *testing.T) { t.Parallel() - db, pubsub := dbtestutil.NewDB(t) - accessURL, err := url.Parse(c.accessURL) require.NoError(t, err) - client := coderdtest.New(t, &coderdtest.Options{ - Database: db, - Pubsub: pubsub, + client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ AccessURL: accessURL, AppHostname: c.appHostname, }) _ = coderdtest.CreateFirstUser(t, client) + // Make sure there's a signing key! + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + + // Make sure there's an encryption key! + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, + }) + // Disable redirects. client.HTTPClient.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 94c8f82726374..956b3ff958aec 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -608,11 +608,8 @@ func TestProxyRegisterDeregister(t *testing.T) { func TestIssueSignedAppToken(t *testing.T) { t.Parallel() - db, pubsub := dbtestutil.NewDB(t) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ - Database: db, - Pubsub: pubsub, IncludeProvisionerDaemon: true, }, LicenseOptions: &coderdenttest.LicenseOptions{ @@ -622,6 +619,10 @@ func TestIssueSignedAppToken(t *testing.T) { }, }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + // Create a workspace + apps authToken := uuid.NewString() version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ @@ -715,6 +716,10 @@ func TestReconnectingPTYSignedToken(t *testing.T) { closer.Close() }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + // Create a workspace + apps authToken := uuid.NewString() version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ @@ -945,16 +950,14 @@ func TestGetCryptoKeys(t *testing.T) { keys, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) require.NotEmpty(t, keys) - // 1 key is generated on startup, the other is the one we generated for our test. - require.Equal(t, 2, len(keys.CryptoKeys)) + require.Equal(t, 1, len(keys.CryptoKeys)) requireContainsKeys(t, keys.CryptoKeys, encryptionKey) - requireNotContainsKeys(t, keys.CryptoKeys, signingKey) keys, err = proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsToken) require.NoError(t, err) require.NotEmpty(t, keys) + require.Equal(t, 1, len(keys.CryptoKeys)) requireContainsKeys(t, keys.CryptoKeys, signingKey) - requireNotContainsKeys(t, keys.CryptoKeys, encryptionKey) }) t.Run("InvalidFeature", func(t *testing.T) { @@ -1027,18 +1030,6 @@ func TestGetCryptoKeys(t *testing.T) { }) } -func requireNotContainsKeys(t *testing.T, keys []codersdk.CryptoKey, unexpected ...codersdk.CryptoKey) { - t.Helper() - - for _, expectedKey := range unexpected { - for _, key := range keys { - if key.Feature == expectedKey.Feature && key.Sequence == expectedKey.Sequence { - t.Fatalf("unexpected key %+v found", expectedKey) - } - } - } -} - func requireContainsKeys(t *testing.T, keys []codersdk.CryptoKey, expected ...codersdk.CryptoKey) { t.Helper() diff --git a/enterprise/workspaceapps_test.go b/enterprise/workspaceapps_test.go index f4ba577f13e33..be1a6737ccc8a 100644 --- a/enterprise/workspaceapps_test.go +++ b/enterprise/workspaceapps_test.go @@ -5,6 +5,9 @@ import ( "testing" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/workspaceapps/apptest" "github.com/coder/coder/v2/codersdk" @@ -36,6 +39,9 @@ func TestWorkspaceApps(t *testing.T) { flushStatsCollectorCh <- flushStatsCollectorDone <-flushStatsCollectorDone } + + db, pubsub := dbtestutil.NewDB(t) + client, _, _, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ Options: &coderdtest.Options{ DeploymentValues: deploymentValues, @@ -51,6 +57,8 @@ func TestWorkspaceApps(t *testing.T) { }, }, WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions, + Database: db, + Pubsub: pubsub, }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ @@ -59,6 +67,13 @@ func TestWorkspaceApps(t *testing.T) { }, }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, + }) + return &apptest.Deployment{ Options: opts, SDKClient: client, diff --git a/enterprise/wsproxy/wsproxy_test.go b/enterprise/wsproxy/wsproxy_test.go index 3d3926c5afae7..4add46af9bc0a 100644 --- a/enterprise/wsproxy/wsproxy_test.go +++ b/enterprise/wsproxy/wsproxy_test.go @@ -25,6 +25,9 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/healthcheck/derphealth" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/workspaceapps/apptest" @@ -932,6 +935,9 @@ func TestWorkspaceProxyWorkspaceApps(t *testing.T) { if opts.PrimaryAppHost == "" { opts.PrimaryAppHost = "*.primary.test.coder.com" } + + db, pubsub := dbtestutil.NewDB(t) + client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ Options: &coderdtest.Options{ DeploymentValues: deploymentValues, @@ -947,6 +953,8 @@ func TestWorkspaceProxyWorkspaceApps(t *testing.T) { }, }, WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions, + Database: db, + Pubsub: pubsub, }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ @@ -959,6 +967,13 @@ func TestWorkspaceProxyWorkspaceApps(t *testing.T) { _ = closer.Close() }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, + }) + // Create the external proxy if opts.DisableSubdomainApps { opts.AppHost = "" @@ -1002,6 +1017,8 @@ func TestWorkspaceProxyWorkspaceApps_BlockDirect(t *testing.T) { if opts.PrimaryAppHost == "" { opts.PrimaryAppHost = "*.primary.test.coder.com" } + + db, pubsub := dbtestutil.NewDB(t) client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{ Options: &coderdtest.Options{ DeploymentValues: deploymentValues, @@ -1017,6 +1034,8 @@ func TestWorkspaceProxyWorkspaceApps_BlockDirect(t *testing.T) { }, }, WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions, + Database: db, + Pubsub: pubsub, }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{ @@ -1029,6 +1048,13 @@ func TestWorkspaceProxyWorkspaceApps_BlockDirect(t *testing.T) { _ = closer.Close() }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, + }) + // Create the external proxy if opts.DisableSubdomainApps { opts.AppHost = "" diff --git a/scaletest/createworkspaces/run_test.go b/scaletest/createworkspaces/run_test.go index 73e26db71970b..d76a6b3f620b8 100644 --- a/scaletest/createworkspaces/run_test.go +++ b/scaletest/createworkspaces/run_test.go @@ -15,6 +15,8 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/agent" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/codersdk" @@ -55,9 +57,12 @@ func Test_Runner(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() - client := coderdtest.New(t, &coderdtest.Options{ + client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ IncludeProvisionerDaemon: true, }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() @@ -343,9 +348,12 @@ func Test_Runner(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() - client := coderdtest.New(t, &coderdtest.Options{ + client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ IncludeProvisionerDaemon: true, }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() diff --git a/scaletest/reconnectingpty/run_test.go b/scaletest/reconnectingpty/run_test.go index 84e2b0abf828f..84b36c7b3d39a 100644 --- a/scaletest/reconnectingpty/run_test.go +++ b/scaletest/reconnectingpty/run_test.go @@ -12,6 +12,9 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/workspacesdk" @@ -248,9 +251,15 @@ func Test_Runner(t *testing.T) { func setupRunnerTest(t *testing.T) (client *codersdk.Client, agentID uuid.UUID) { t.Helper() + db, pubsub := dbtestutil.NewDB(t) client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Database: db, + Pubsub: pubsub, IncludeProvisionerDaemon: true, }) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() diff --git a/scaletest/workspacetraffic/run_test.go b/scaletest/workspacetraffic/run_test.go index bb9d88b969d58..820e3202b6f57 100644 --- a/scaletest/workspacetraffic/run_test.go +++ b/scaletest/workspacetraffic/run_test.go @@ -17,6 +17,8 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisioner/echo" "github.com/coder/coder/v2/provisionersdk/proto" @@ -42,7 +44,10 @@ func TestRun(t *testing.T) { t.Parallel() // We need to stand up an in-memory coderd and run a fake workspace. var ( - client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) + _ = dbgen.CryptoKey(t, db, database.CryptoKey{ + Feature: database.CryptoKeyFeatureWorkspaceAppsToken, + }) firstUser = coderdtest.CreateFirstUser(t, client) authToken = uuid.NewString() agentName = "agent" From 6d3c10350e9f0446e23b89d1adfd9442abf90a93 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sun, 20 Oct 2024 18:51:18 +0000 Subject: [PATCH 24/39] pr comments --- cli/server.go | 15 +++--- cli/server_test.go | 39 --------------- coderd/coderd.go | 21 +++++++-- coderd/coderdtest/coderdtest.go | 6 +++ coderd/cryptokeys/cache.go | 6 ++- coderd/cryptokeys/rotate.go | 2 +- coderd/jwtutils/jws.go | 15 ++++++ coderd/userauth.go | 4 ++ coderd/userauth_test.go | 7 +-- coderd/workspaceapps.go | 6 ++- coderd/workspaceapps/db.go | 22 ++++++--- coderd/workspaceapps/db_test.go | 4 +- coderd/workspaceapps/request.go | 2 +- coderd/workspaceapps/token.go | 23 ++++++++- coderd/workspaceapps_test.go | 47 ++++++++++++++----- .../workspacesdk/connector_internal_test.go | 2 +- tailnet/resume.go | 2 +- tailnet/resume_test.go | 2 +- 18 files changed, 133 insertions(+), 92 deletions(-) diff --git a/cli/server.go b/cli/server.go index ebbb9806a92c3..2c1f8fab10c1d 100644 --- a/cli/server.go +++ b/cli/server.go @@ -746,27 +746,24 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. return xerrors.Errorf("set deployment id: %w", err) } - // Start a background process that rotates keys. - err = cryptokeys.StartRotator(ctx, options.Logger.Named("keyrotator"), options.Database) - if err != nil { - return xerrors.Errorf("start key rotator: %w", err) - } - fetcher := &cryptokeys.DBFetcher{ DB: options.Database, } - // TODO(JonA): The instantiation of this cache + coordinator seems like it should be done inside coderd so that it uses the correct context. resumeKeycache, err := cryptokeys.NewSigningCache(ctx, logger, fetcher, codersdk.CryptoKeyFeatureTailnetResume, ) if err != nil { - return xerrors.Errorf("create tailnet resume key cache: %w", err) + logger.Critical(ctx, "failed to properly instantiate tailnet resume signing cache", slog.Error(err)) } - options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider(resumeKeycache, quartz.NewReal(), tailnet.DefaultResumeTokenExpiry) + options.CoordinatorResumeTokenProvider = tailnet.NewResumeTokenKeyProvider( + resumeKeycache, + quartz.NewReal(), + tailnet.DefaultResumeTokenExpiry, + ) options.RuntimeConfig = runtimeconfig.NewManager() diff --git a/cli/server_test.go b/cli/server_test.go index 6fc083d43bf60..ad6a98038c7bb 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -44,7 +44,6 @@ import ( "github.com/coder/coder/v2/cli/clitest" "github.com/coder/coder/v2/cli/config" "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/telemetry" @@ -1591,44 +1590,6 @@ func TestServer(t *testing.T) { w.RequireSuccess() }) }) - - t.Run("CryptoKeysGenerated", func(t *testing.T) { - t.Parallel() - if testing.Short() { - t.SkipNow() - } - - if runtime.GOOS != "linux" || testing.Short() { - // Skip on non-Linux because it spawns a PostgreSQL instance. - t.SkipNow() - } - connectionURL, closeFunc, err := dbtestutil.Open() - require.NoError(t, err) - defer closeFunc() - - inv, cfg := clitest.New(t, - "server", - "--http-address", ":0", - "--access-url", "http://example.com", - "--postgres-url", connectionURL, - "--cache-dir", t.TempDir(), - ) - - const superDuperLong = testutil.WaitSuperLong * 3 - ctx := testutil.Context(t, superDuperLong) - clitest.Start(t, inv.WithContext(ctx)) - _ = waitAccessURL(t, cfg) - - logger := slogtest.Make(t, nil) - sqldb, err := cli.ConnectToPostgres(ctx, logger, "postgres", connectionURL) - require.NoError(t, err) - defer sqldb.Close() - - db := database.New(sqldb) - keys, err := db.GetCryptoKeys(ctx) - require.NoError(t, err) - require.Len(t, keys, len(database.AllCryptoKeyFeatureValues())) - }) } func TestServer_Production(t *testing.T) { diff --git a/coderd/coderd.go b/coderd/coderd.go index 351c57aa21175..4c1e555ac4477 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -186,7 +186,6 @@ type Options struct { TemplateScheduleStore *atomic.Pointer[schedule.TemplateScheduleStore] UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore] AccessControlStore *atomic.Pointer[dbauthz.AccessControlStore] - // workspace applications. It consists of both a signing and encryption key. // CoordinatorResumeTokenProvider is used to provide and validate resume // tokens issued by and passed to the coordinator DRPC API. CoordinatorResumeTokenProvider tailnet.ResumeTokenProvider @@ -255,6 +254,7 @@ type Options struct { AppSigningKeyCache cryptokeys.SigningKeycache AppEncryptionKeyCache cryptokeys.EncryptionKeycache OIDCConvertKeyCache cryptokeys.SigningKeycache + Clock quartz.Clock } // @title Coder API @@ -356,6 +356,9 @@ func New(options *Options) *API { if options.PrometheusRegistry == nil { options.PrometheusRegistry = prometheus.NewRegistry() } + if options.Clock == nil { + options.Clock = quartz.NewReal() + } if options.DERPServer == nil && options.DeploymentValues.DERP.Server.Enable { options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp"))) } @@ -459,7 +462,7 @@ func New(options *Options) *API { fetcher, codersdk.CryptoKeyFeatureOIDCConvert, ) - must(options.Logger, "start oidc convert key cache", err) + options.Logger.Critical(ctx, "failed to properly instantiate oidc convert signing cache", slog.Error(err)) } if options.AppSigningKeyCache == nil { @@ -468,7 +471,7 @@ func New(options *Options) *API { fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsToken, ) - must(options.Logger, "start app signing key cache", err) + options.Logger.Critical(ctx, "failed to properly instantiate app signing key cache", slog.Error(err)) } if options.AppEncryptionKeyCache == nil { @@ -477,7 +480,15 @@ func New(options *Options) *API { fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey, ) - must(options.Logger, "start app encryption key cache", err) + options.Logger.Critical(ctx, "failed to properly instantiate app encryption key cache", slog.Error(err)) + } + + // Start a background process that rotates keys. We intentionally start this after the caches + // are created to force initial requests for a key to populate the caches. This helps catch + // bugs that may only occur when a key isn't precached in tests and the latency cost is minimal. + err = cryptokeys.StartRotator(ctx, options.Logger, options.Database) + if err != nil { + must(options.Logger, "failed to start key rotator", err) } api := &API{ @@ -642,7 +653,7 @@ func New(options *Options) *API { ResumeTokenProvider: api.Options.CoordinatorResumeTokenProvider, }) if err != nil { - api.Logger.Fatal(api.ctx, "failed to initialize tailnet client service", slog.Error(err)) + must(api.Logger, "failed to initialize tailnet client service", err) } api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index cc6611d766faf..259971f7586c6 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -55,6 +55,7 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/autobuild" "github.com/coder/coder/v2/coderd/awsidentity" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" "github.com/coder/coder/v2/coderd/database/dbauthz" @@ -88,6 +89,7 @@ import ( sdkproto "github.com/coder/coder/v2/provisionersdk/proto" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" ) type Options struct { @@ -158,6 +160,8 @@ type Options struct { WorkspaceUsageTrackerFlush chan int WorkspaceUsageTrackerTick chan time.Time NotificationsEnqueuer notifications.Enqueuer + APIKeyEncryptionCache cryptokeys.EncryptionKeycache + Clock quartz.Clock } // New constructs a codersdk client connected to an in-memory API instance. @@ -532,6 +536,8 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can WorkspaceUsageTracker: wuTracker, NotificationsEnqueuer: options.NotificationsEnqueuer, OneTimePasscodeValidityPeriod: options.OneTimePasscodeValidityPeriod, + Clock: options.Clock, + AppEncryptionKeyCache: options.APIKeyEncryptionCache, } } diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index 0f4e88daef357..e4ab9cde7e625 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -302,14 +302,15 @@ func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, [] func (c *cache) refresh() { now := c.clock.Now("CryptoKeyCache", "refresh") c.mu.Lock() - defer c.mu.Unlock() if c.closed { + c.mu.Unlock() return } // If something's already fetching, we don't need to do anything. if c.fetching { + c.mu.Unlock() return } @@ -317,6 +318,7 @@ func (c *cache) refresh() { // is ongoing but prior to the timer getting reset. In this case we want to // avoid double fetching. if now.Sub(c.lastFetch) < refreshInterval { + c.mu.Unlock() return } @@ -329,8 +331,8 @@ func (c *cache) refresh() { return } - // We don't defer an unlock here due to the deferred unlock at the top of the function. c.mu.Lock() + defer c.mu.Unlock() c.lastFetch = c.clock.Now() c.refresher.Reset(refreshInterval) diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index 186285c4a21c7..090100fd8bf32 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -59,7 +59,7 @@ func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, op ctx = dbauthz.AsKeyRotator(ctx) kr := &rotator{ db: db, - logger: logger, + logger: logger.Named("keyrotator"), clock: quartz.NewReal(), keyDuration: DefaultKeyDuration, features: database.AllCryptoKeyFeatureValues(), diff --git a/coderd/jwtutils/jws.go b/coderd/jwtutils/jws.go index 553d0da0aeb2d..4d60404e38ece 100644 --- a/coderd/jwtutils/jws.go +++ b/coderd/jwtutils/jws.go @@ -14,6 +14,21 @@ const ( keyIDHeaderKey = "kid" ) +// RegisteredClaims is a convenience type for embedding jwt.Claims. It should be +// preferred over embedding jwt.Claims directly since it will ensure that certain fields are set. +type RegisteredClaims jwt.Claims + +func (r RegisteredClaims) Validate(e jwt.Expected) error { + if r.Expiry == nil { + return xerrors.Errorf("expiry is required") + } + if e.Time.IsZero() { + return xerrors.Errorf("expected time is required") + } + + return (jwt.Claims(r)).Validate(e) +} + // Claims defines the payload for a JWT. Most callers // should embed jwt.Claims type Claims interface { diff --git a/coderd/userauth.go b/coderd/userauth.go index eaf2aa3659ae8..187e970cf8119 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -60,6 +60,10 @@ type OAuthConvertStateClaims struct { ToLoginType codersdk.LoginType `json:"to_login_type"` } +func (o *OAuthConvertStateClaims) Validate(e jwt.Expected) error { + return o.Claims.Validate(e) +} + // postConvertLoginType replies with an oauth state token capable of converting // the user to an oauth user. // diff --git a/coderd/userauth_test.go b/coderd/userauth_test.go index df33ce2fedd96..20dfe7f723899 100644 --- a/coderd/userauth_test.go +++ b/coderd/userauth_test.go @@ -1309,16 +1309,11 @@ func TestUserOIDC(t *testing.T) { cfg.AllowSignups = true }) - client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ + client := coderdtest.New(t, &coderdtest.Options{ Auditor: auditor, OIDCConfig: cfg, }) - // Make sure there's a signing key! - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureOIDCConvert, - }) - owner := coderdtest.CreateFirstUser(t, client) user, userData := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID) diff --git a/coderd/workspaceapps.go b/coderd/workspaceapps.go index 52bbcd3b0b70d..e264dbd80b58d 100644 --- a/coderd/workspaceapps.go +++ b/coderd/workspaceapps.go @@ -123,9 +123,11 @@ func (api *API) workspaceApplicationAuth(rw http.ResponseWriter, r *http.Request return } - encryptedAPIKey, err := jwtutils.Encrypt(ctx, api.AppEncryptionKeyCache, workspaceapps.EncryptedAPIKeyPayload{ + payload := workspaceapps.EncryptedAPIKeyPayload{ APIKey: cookie.Value, - }) + } + payload.Fill(api.Clock.Now()) + encryptedAPIKey, err := jwtutils.Encrypt(ctx, api.AppEncryptionKeyCache, payload) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ Message: "Failed to encrypt API key.", diff --git a/coderd/workspaceapps/db.go b/coderd/workspaceapps/db.go index b3f223dec9ebd..1aa4dfe91bdd0 100644 --- a/coderd/workspaceapps/db.go +++ b/coderd/workspaceapps/db.go @@ -39,12 +39,20 @@ type DBTokenProvider struct { DeploymentValues *codersdk.DeploymentValues OAuth2Configs *httpmw.OAuth2Configs WorkspaceAgentInactiveTimeout time.Duration - TokenSigner cryptokeys.SigningKeycache + Keycache cryptokeys.SigningKeycache } var _ SignedTokenProvider = &DBTokenProvider{} -func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authorizer, db database.Store, cfg *codersdk.DeploymentValues, oauth2Cfgs *httpmw.OAuth2Configs, workspaceAgentInactiveTimeout time.Duration, signer cryptokeys.SigningKeycache) SignedTokenProvider { +func NewDBTokenProvider(log slog.Logger, + accessURL *url.URL, + authz rbac.Authorizer, + db database.Store, + cfg *codersdk.DeploymentValues, + oauth2Cfgs *httpmw.OAuth2Configs, + workspaceAgentInactiveTimeout time.Duration, + signer cryptokeys.SigningKeycache, +) SignedTokenProvider { if workspaceAgentInactiveTimeout == 0 { workspaceAgentInactiveTimeout = 1 * time.Minute } @@ -57,12 +65,12 @@ func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authoriz DeploymentValues: cfg, OAuth2Configs: oauth2Cfgs, WorkspaceAgentInactiveTimeout: workspaceAgentInactiveTimeout, - TokenSigner: signer, + Keycache: signer, } } func (p *DBTokenProvider) FromRequest(r *http.Request) (*SignedToken, bool) { - return FromRequest(r, p.TokenSigner) + return FromRequest(r, p.Keycache) } func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *http.Request, issueReq IssueTokenRequest) (*SignedToken, string, bool) { @@ -214,11 +222,11 @@ func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r * return nil, "", false } - // Sign the token. - token.Claims = jwt.Claims{ + token.RegisteredClaims = jwtutils.RegisteredClaims{ Expiry: jwt.NewNumericDate(time.Now().Add(DefaultTokenExpiry)), } - tokenStr, err := jwtutils.Sign(ctx, p.TokenSigner, token) + // Sign the token. + tokenStr, err := jwtutils.Sign(ctx, p.Keycache, token) if err != nil { WriteWorkspaceApp500(p.Logger, p.DashboardURL, rw, r, &appReq, err, "generate token") return nil, "", false diff --git a/coderd/workspaceapps/db_test.go b/coderd/workspaceapps/db_test.go index 48709daedcb2a..b48372199dd28 100644 --- a/coderd/workspaceapps/db_test.go +++ b/coderd/workspaceapps/db_test.go @@ -287,7 +287,7 @@ func Test_ResolveRequest(t *testing.T) { _ = w.Body.Close() require.Equal(t, &workspaceapps.SignedToken{ - Claims: jwt.Claims{ + RegisteredClaims: jwtutils.RegisteredClaims{ Expiry: jwt.NewNumericDate(token.Expiry.Time()), }, Request: req, @@ -554,7 +554,7 @@ func Test_ResolveRequest(t *testing.T) { // App name differs AppSlugOrPort: appNamePublic, }).Normalize(), - Claims: jwt.Claims{ + RegisteredClaims: jwtutils.RegisteredClaims{ Expiry: jwt.NewNumericDate(time.Now().Add(time.Minute)), }, UserID: me.ID, diff --git a/coderd/workspaceapps/request.go b/coderd/workspaceapps/request.go index ade95c782c5d9..0833ab731fe67 100644 --- a/coderd/workspaceapps/request.go +++ b/coderd/workspaceapps/request.go @@ -124,7 +124,7 @@ func (r Request) Normalize() Request { return req } -// Validate ensures the request is correct and contains the necessary +// Check ensures the request is correct and contains the necessary // parameters. func (r Request) Check() error { switch r.AccessMethod { diff --git a/coderd/workspaceapps/token.go b/coderd/workspaceapps/token.go index 08e2df22b7269..dcd8c5a0e5c34 100644 --- a/coderd/workspaceapps/token.go +++ b/coderd/workspaceapps/token.go @@ -7,6 +7,7 @@ import ( "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" + "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" @@ -17,7 +18,7 @@ import ( // contains the details of the workspace app that the token is valid for to // avoid database queries. type SignedToken struct { - jwt.Claims + jwtutils.RegisteredClaims // Request details. Request `json:"request"` @@ -49,10 +50,28 @@ func (t SignedToken) MatchesRequest(req Request) bool { } type EncryptedAPIKeyPayload struct { - jwt.Claims + jwtutils.RegisteredClaims APIKey string `json:"api_key"` } +func (e *EncryptedAPIKeyPayload) Fill(now time.Time) { + e.Issuer = "coderd" + e.Audience = jwt.Audience{"wsproxy"} + e.Expiry = jwt.NewNumericDate(now.Add(time.Minute)) + e.NotBefore = jwt.NewNumericDate(now.Add(-time.Minute)) +} + +func (e EncryptedAPIKeyPayload) Validate(ex jwt.Expected) error { + if e.NotBefore == nil { + return xerrors.Errorf("not before is required") + } + + ex.Issuer = "coderd" + ex.AnyAudience = jwt.Audience{"wsproxy"} + + return e.RegisteredClaims.Validate(ex) +} + // FromRequest returns the signed token from the request, if it exists and is // valid. The caller must check that the token matches the request. func FromRequest(r *http.Request, mgr cryptokeys.SigningKeycache) (*SignedToken, bool) { diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index a5cec65410303..dc7d747ebe36e 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -5,15 +5,22 @@ import ( "net/http" "net/url" "testing" + "time" + "cdr.dev/slog/sloggers/slogtest" + "github.com/go-jose/go-jose/v4/jwt" "github.com/stretchr/testify/require" "github.com/coder/coder/v2/coderd/coderdtest" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" ) func TestGetAppHost(t *testing.T) { @@ -180,24 +187,30 @@ func TestWorkspaceApplicationAuth(t *testing.T) { t.Run(c.name, func(t *testing.T) { t.Parallel() + ctx := testutil.Context(t, testutil.WaitMedium) + logger := slogtest.Make(t, nil) accessURL, err := url.Parse(c.accessURL) require.NoError(t, err) - client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ - AccessURL: accessURL, - AppHostname: c.appHostname, - }) - _ = coderdtest.CreateFirstUser(t, client) + db, ps := dbtestutil.NewDB(t) + fetcher := &cryptokeys.DBFetcher{ + DB: db, + } - // Make sure there's a signing key! - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) + kc, err := cryptokeys.NewEncryptionCache(ctx, logger, fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) + require.NoError(t, err) + + clock := quartz.NewMock(t) - // Make sure there's an encryption key! - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, + client := coderdtest.New(t, &coderdtest.Options{ + AccessURL: accessURL, + AppHostname: c.appHostname, + Database: db, + Pubsub: ps, + APIKeyEncryptionCache: kc, + Clock: clock, }) + _ = coderdtest.CreateFirstUser(t, client) // Disable redirects. client.HTTPClient.CheckRedirect = func(_ *http.Request, _ []*http.Request) error { @@ -245,7 +258,15 @@ func TestWorkspaceApplicationAuth(t *testing.T) { loc.RawQuery = q.Encode() require.Equal(t, c.expectRedirect, loc.String()) - // The decrypted key is verified in the apptest test suite. + var token workspaceapps.EncryptedAPIKeyPayload + err = jwtutils.Decrypt(ctx, kc, encryptedAPIKey, &token, jwtutils.WithDecryptExpected(jwt.Expected{ + Time: clock.Now(), + AnyAudience: jwt.Audience{"wsproxy"}, + Issuer: "coderd", + })) + require.NoError(t, err) + require.Equal(t, jwt.NewNumericDate(clock.Now().Add(time.Minute)), token.Expiry) + require.Equal(t, jwt.NewNumericDate(clock.Now().Add(-time.Minute)), token.NotBefore) }) } } diff --git a/codersdk/workspacesdk/connector_internal_test.go b/codersdk/workspacesdk/connector_internal_test.go index 2d70278095653..1443f93cf7a55 100644 --- a/codersdk/workspacesdk/connector_internal_test.go +++ b/codersdk/workspacesdk/connector_internal_test.go @@ -167,7 +167,7 @@ func TestTailnetAPIConnector_ResumeToken(t *testing.T) { resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) mgr := cryptokeys.StaticKey{ - ID: uuid.New().String(), + ID: "123", Key: resumeTokenSigningKey[:], } resumeTokenProvider := tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour) diff --git a/tailnet/resume.go b/tailnet/resume.go index 6b4ea2417b3da..0201553f10316 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -70,7 +70,7 @@ func NewResumeTokenKeyProvider(key cryptokeys.SigningKeycache, clock quartz.Cloc func (p ResumeTokenKeyProvider) GenerateResumeToken(ctx context.Context, peerID uuid.UUID) (*proto.RefreshResumeTokenResponse, error) { exp := p.clock.Now().Add(p.expiry) - payload := jwt.Claims{ + payload := jwtutils.RegisteredClaims{ Subject: peerID.String(), Expiry: jwt.NewNumericDate(exp), } diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index 6c25c12eeaf36..80f2fc175dc22 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -93,7 +93,7 @@ func TestResumeTokenKeyProvider(t *testing.T) { func newKeySigner(key tailnet.ResumeTokenSigningKey) cryptokeys.StaticKey { return cryptokeys.StaticKey{ - ID: uuid.New().String(), + ID: "123", Key: key[:], } } From bfa88b7b9a8272ee00493acf49a5b32805a01d7f Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sun, 20 Oct 2024 19:01:43 +0000 Subject: [PATCH 25/39] Refactor tests to remove direct database setup Refactored various tests to eliminate the need for manually setting up a database connection and generating crypto keys. This change simplifies the test setup and makes it less dependent on database-specific configurations, aligning with recent structural refactoring. --- coderd/workspaceapps/db_test.go | 10 ---------- coderd/workspaceapps/token_test.go | 4 ++-- enterprise/workspaceapps_test.go | 9 --------- scaletest/createworkspaces/run_test.go | 12 ++---------- scaletest/reconnectingpty/run_test.go | 9 --------- scaletest/workspacetraffic/run_test.go | 7 +------ 6 files changed, 5 insertions(+), 46 deletions(-) diff --git a/coderd/workspaceapps/db_test.go b/coderd/workspaceapps/db_test.go index b48372199dd28..bf364f1ce62b3 100644 --- a/coderd/workspaceapps/db_test.go +++ b/coderd/workspaceapps/db_test.go @@ -20,9 +20,6 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" - "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" @@ -79,7 +76,6 @@ func Test_ResolveRequest(t *testing.T) { deploymentValues.Dangerous.AllowPathAppSharing = true deploymentValues.Dangerous.AllowPathAppSiteOwnerAccess = true - db, pubsub := dbtestutil.NewDB(t) client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ AppHostname: "*.test.coder.com", DeploymentValues: deploymentValues, @@ -95,17 +91,11 @@ func Test_ResolveRequest(t *testing.T) { "CF-Connecting-IP", }, }, - Database: db, - Pubsub: pubsub, }) t.Cleanup(func() { _ = closer.Close() }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) - ctx := testutil.Context(t, testutil.WaitMedium) firstUser := coderdtest.CreateFirstUser(t, client) diff --git a/coderd/workspaceapps/token_test.go b/coderd/workspaceapps/token_test.go index 5bd7abeb29637..91f2ef69aa6b2 100644 --- a/coderd/workspaceapps/token_test.go +++ b/coderd/workspaceapps/token_test.go @@ -301,7 +301,7 @@ func Test_FromRequest(t *testing.T) { signer := newSigner(t) token := workspaceapps.SignedToken{ - Claims: jwt.Claims{ + RegisteredClaims: jwtutils.RegisteredClaims{ Expiry: jwt.NewNumericDate(time.Now().Add(time.Hour)), }, Request: workspaceapps.Request{ @@ -321,7 +321,7 @@ func Test_FromRequest(t *testing.T) { // Add an expired cookie expired := token - expired.Claims.Expiry = jwt.NewNumericDate(time.Now().Add(time.Hour * -1)) + expired.RegisteredClaims.Expiry = jwt.NewNumericDate(time.Now().Add(time.Hour * -1)) expiredStr, err := jwtutils.Sign(ctx, signer, expired) require.NoError(t, err) r.AddCookie(&http.Cookie{ diff --git a/enterprise/workspaceapps_test.go b/enterprise/workspaceapps_test.go index be1a6737ccc8a..51d0314c45767 100644 --- a/enterprise/workspaceapps_test.go +++ b/enterprise/workspaceapps_test.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpmw" "github.com/coder/coder/v2/coderd/workspaceapps/apptest" @@ -67,13 +65,6 @@ func TestWorkspaceApps(t *testing.T) { }, }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsAPIKey, - }) - return &apptest.Deployment{ Options: opts, SDKClient: client, diff --git a/scaletest/createworkspaces/run_test.go b/scaletest/createworkspaces/run_test.go index d76a6b3f620b8..73e26db71970b 100644 --- a/scaletest/createworkspaces/run_test.go +++ b/scaletest/createworkspaces/run_test.go @@ -15,8 +15,6 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/agent" "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/util/ptr" "github.com/coder/coder/v2/codersdk" @@ -57,12 +55,9 @@ func Test_Runner(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() - client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ + client := coderdtest.New(t, &coderdtest.Options{ IncludeProvisionerDaemon: true, }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() @@ -348,12 +343,9 @@ func Test_Runner(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() - client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{ + client := coderdtest.New(t, &coderdtest.Options{ IncludeProvisionerDaemon: true, }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() diff --git a/scaletest/reconnectingpty/run_test.go b/scaletest/reconnectingpty/run_test.go index 84b36c7b3d39a..84e2b0abf828f 100644 --- a/scaletest/reconnectingpty/run_test.go +++ b/scaletest/reconnectingpty/run_test.go @@ -12,9 +12,6 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" - "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/workspacesdk" @@ -251,15 +248,9 @@ func Test_Runner(t *testing.T) { func setupRunnerTest(t *testing.T) (client *codersdk.Client, agentID uuid.UUID) { t.Helper() - db, pubsub := dbtestutil.NewDB(t) client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ - Database: db, - Pubsub: pubsub, IncludeProvisionerDaemon: true, }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) user := coderdtest.CreateFirstUser(t, client) authToken := uuid.NewString() diff --git a/scaletest/workspacetraffic/run_test.go b/scaletest/workspacetraffic/run_test.go index 820e3202b6f57..bb9d88b969d58 100644 --- a/scaletest/workspacetraffic/run_test.go +++ b/scaletest/workspacetraffic/run_test.go @@ -17,8 +17,6 @@ import ( "github.com/coder/coder/v2/agent/agenttest" "github.com/coder/coder/v2/coderd/coderdtest" - "github.com/coder/coder/v2/coderd/database" - "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisioner/echo" "github.com/coder/coder/v2/provisionersdk/proto" @@ -44,10 +42,7 @@ func TestRun(t *testing.T) { t.Parallel() // We need to stand up an in-memory coderd and run a fake workspace. var ( - client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) + client = coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) firstUser = coderdtest.CreateFirstUser(t, client) authToken = uuid.NewString() agentName = "agent" From 495c28f4d4979400a2c27b015e2f31eea12b7f05 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sun, 20 Oct 2024 19:06:58 +0000 Subject: [PATCH 26/39] Rename cryptokey migration files to update sequence --- ...tokey_features.down.sql => 000267_cryptokey_features.down.sql} | 0 ...cryptokey_features.up.sql => 000267_cryptokey_features.up.sql} | 0 ...cryptokey_features.up.sql => 000267_cryptokey_features.up.sql} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000266_cryptokey_features.down.sql => 000267_cryptokey_features.down.sql} (100%) rename coderd/database/migrations/{000266_cryptokey_features.up.sql => 000267_cryptokey_features.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000266_cryptokey_features.up.sql => 000267_cryptokey_features.up.sql} (100%) diff --git a/coderd/database/migrations/000266_cryptokey_features.down.sql b/coderd/database/migrations/000267_cryptokey_features.down.sql similarity index 100% rename from coderd/database/migrations/000266_cryptokey_features.down.sql rename to coderd/database/migrations/000267_cryptokey_features.down.sql diff --git a/coderd/database/migrations/000266_cryptokey_features.up.sql b/coderd/database/migrations/000267_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/000266_cryptokey_features.up.sql rename to coderd/database/migrations/000267_cryptokey_features.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000266_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000267_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000266_cryptokey_features.up.sql rename to coderd/database/migrations/testdata/fixtures/000267_cryptokey_features.up.sql From 692bb36025eb7bc3982f0e4fc5add01014744efa Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sun, 20 Oct 2024 19:08:40 +0000 Subject: [PATCH 27/39] Fix conditional logging in key cache initialization Ensure that logging only occurs when an error is present during the initialization of the key caches. This prevents unnecessary log entries when no error is encountered, improving log clarity and reducing noise. --- coderd/coderd.go | 12 +++++++++--- coderd/workspaceapps_test.go | 3 ++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 4c1e555ac4477..ae23d15d6b239 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -462,7 +462,9 @@ func New(options *Options) *API { fetcher, codersdk.CryptoKeyFeatureOIDCConvert, ) - options.Logger.Critical(ctx, "failed to properly instantiate oidc convert signing cache", slog.Error(err)) + if err != nil { + options.Logger.Critical(ctx, "failed to properly instantiate oidc convert signing cache", slog.Error(err)) + } } if options.AppSigningKeyCache == nil { @@ -471,7 +473,9 @@ func New(options *Options) *API { fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsToken, ) - options.Logger.Critical(ctx, "failed to properly instantiate app signing key cache", slog.Error(err)) + if err != nil { + options.Logger.Critical(ctx, "failed to properly instantiate app signing key cache", slog.Error(err)) + } } if options.AppEncryptionKeyCache == nil { @@ -480,7 +484,9 @@ func New(options *Options) *API { fetcher, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey, ) - options.Logger.Critical(ctx, "failed to properly instantiate app encryption key cache", slog.Error(err)) + if err != nil { + options.Logger.Critical(ctx, "failed to properly instantiate app encryption key cache", slog.Error(err)) + } } // Start a background process that rotates keys. We intentionally start this after the caches diff --git a/coderd/workspaceapps_test.go b/coderd/workspaceapps_test.go index dc7d747ebe36e..52b3e18b4e6ad 100644 --- a/coderd/workspaceapps_test.go +++ b/coderd/workspaceapps_test.go @@ -7,10 +7,11 @@ import ( "testing" "time" - "cdr.dev/slog/sloggers/slogtest" "github.com/go-jose/go-jose/v4/jwt" "github.com/stretchr/testify/require" + "cdr.dev/slog/sloggers/slogtest" + "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" From 4028995f96cdadfa97ddbb4973b611525ee6713a Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Sun, 20 Oct 2024 19:21:53 +0000 Subject: [PATCH 28/39] Refactor crypto key management in tests Remove redundant database setup for tests by generating keys in memory. Enhance tests by validating both inclusion and exclusion of specific keys. --- enterprise/coderd/workspaceproxy_test.go | 26 +++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go index 956b3ff958aec..0be112b532b7a 100644 --- a/enterprise/coderd/workspaceproxy_test.go +++ b/enterprise/coderd/workspaceproxy_test.go @@ -608,7 +608,7 @@ func TestProxyRegisterDeregister(t *testing.T) { func TestIssueSignedAppToken(t *testing.T) { t.Parallel() - client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ + client, user := coderdenttest.New(t, &coderdenttest.Options{ Options: &coderdtest.Options{ IncludeProvisionerDaemon: true, }, @@ -619,10 +619,6 @@ func TestIssueSignedAppToken(t *testing.T) { }, }) - _ = dbgen.CryptoKey(t, db, database.CryptoKey{ - Feature: database.CryptoKeyFeatureWorkspaceAppsToken, - }) - // Create a workspace + apps authToken := uuid.NewString() version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ @@ -950,14 +946,18 @@ func TestGetCryptoKeys(t *testing.T) { keys, err := proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsAPIKey) require.NoError(t, err) require.NotEmpty(t, keys) - require.Equal(t, 1, len(keys.CryptoKeys)) + // 1 key is generated on startup, the other we manually generated. + require.Equal(t, 2, len(keys.CryptoKeys)) requireContainsKeys(t, keys.CryptoKeys, encryptionKey) + requireNotContainsKeys(t, keys.CryptoKeys, signingKey) keys, err = proxy.SDKClient.CryptoKeys(ctx, codersdk.CryptoKeyFeatureWorkspaceAppsToken) require.NoError(t, err) require.NotEmpty(t, keys) - require.Equal(t, 1, len(keys.CryptoKeys)) + // 1 key is generated on startup, the other we manually generated. + require.Equal(t, 2, len(keys.CryptoKeys)) requireContainsKeys(t, keys.CryptoKeys, signingKey) + requireNotContainsKeys(t, keys.CryptoKeys, encryptionKey) }) t.Run("InvalidFeature", func(t *testing.T) { @@ -1030,6 +1030,18 @@ func TestGetCryptoKeys(t *testing.T) { }) } +func requireNotContainsKeys(t *testing.T, keys []codersdk.CryptoKey, unexpected ...codersdk.CryptoKey) { + t.Helper() + + for _, unexpectedKey := range unexpected { + for _, key := range keys { + if key.Feature == unexpectedKey.Feature && key.Sequence == unexpectedKey.Sequence { + t.Fatalf("unexpected key %+v found", unexpectedKey) + } + } + } +} + func requireContainsKeys(t *testing.T, keys []codersdk.CryptoKey, expected ...codersdk.CryptoKey) { t.Helper() From 6b9a3e4efd3421e36908a5bd5101fd8b2aa67e29 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 21 Oct 2024 21:42:30 +0000 Subject: [PATCH 29/39] add test for oidc jwt --- coderd/coderdtest/coderdtest.go | 2 + coderd/jwtutils/jwe.go | 2 +- coderd/jwtutils/jws.go | 4 +- coderd/userauth.go | 8 +-- coderd/userauth_test.go | 124 +++++++++++++++++++++++++++++++- 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 259971f7586c6..d94a6fbe93c4e 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -161,6 +161,7 @@ type Options struct { WorkspaceUsageTrackerTick chan time.Time NotificationsEnqueuer notifications.Enqueuer APIKeyEncryptionCache cryptokeys.EncryptionKeycache + OIDCConvertKeyCache cryptokeys.SigningKeycache Clock quartz.Clock } @@ -538,6 +539,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can OneTimePasscodeValidityPeriod: options.OneTimePasscodeValidityPeriod, Clock: options.Clock, AppEncryptionKeyCache: options.APIKeyEncryptionCache, + OIDCConvertKeyCache: options.OIDCConvertKeyCache, } } diff --git a/coderd/jwtutils/jwe.go b/coderd/jwtutils/jwe.go index 704dc3d4011d6..bc9d0ddd2a9c8 100644 --- a/coderd/jwtutils/jwe.go +++ b/coderd/jwtutils/jwe.go @@ -106,7 +106,7 @@ func Decrypt(ctx context.Context, d DecryptKeyProvider, token string, claims Cla kid := object.Header.KeyID if kid == "" { - return xerrors.Errorf("expected %q header to be a string", keyIDHeaderKey) + return ErrMissingKeyID } key, err := d.DecryptingKey(ctx, kid) diff --git a/coderd/jwtutils/jws.go b/coderd/jwtutils/jws.go index 4d60404e38ece..f6a9964924424 100644 --- a/coderd/jwtutils/jws.go +++ b/coderd/jwtutils/jws.go @@ -10,6 +10,8 @@ import ( "golang.org/x/xerrors" ) +var ErrMissingKeyID = xerrors.New("missing key ID") + const ( keyIDHeaderKey = "kid" ) @@ -126,7 +128,7 @@ func Verify(ctx context.Context, v VerifyKeyProvider, token string, claims Claim kid := signature.Header.KeyID if kid == "" { - return xerrors.Errorf("expected %q header to be a string", keyIDHeaderKey) + return ErrMissingKeyID } key, err := v.VerifyingKey(ctx, kid) diff --git a/coderd/userauth.go b/coderd/userauth.go index 187e970cf8119..11248fa566f7a 100644 --- a/coderd/userauth.go +++ b/coderd/userauth.go @@ -52,7 +52,7 @@ const ( ) type OAuthConvertStateClaims struct { - jwt.Claims + jwtutils.RegisteredClaims UserID uuid.UUID `json:"user_id"` State string `json:"state"` @@ -61,7 +61,7 @@ type OAuthConvertStateClaims struct { } func (o *OAuthConvertStateClaims) Validate(e jwt.Expected) error { - return o.Claims.Validate(e) + return o.RegisteredClaims.Validate(e) } // postConvertLoginType replies with an oauth state token capable of converting @@ -156,7 +156,7 @@ func (api *API) postConvertLoginType(rw http.ResponseWriter, r *http.Request) { // Eg: Developers with more than 1 deployment. now := time.Now() claims := &OAuthConvertStateClaims{ - Claims: jwt.Claims{ + RegisteredClaims: jwtutils.RegisteredClaims{ Issuer: api.DeploymentID, Subject: stateString, Audience: []string{user.ID.String()}, @@ -1682,7 +1682,7 @@ func (api *API) convertUserToOauth(ctx context.Context, r *http.Request, db data var claims OAuthConvertStateClaims err = jwtutils.Verify(ctx, api.OIDCConvertKeyCache, jwtCookie.Value, &claims) - if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) { + if xerrors.Is(err, cryptokeys.ErrKeyNotFound) || xerrors.Is(err, cryptokeys.ErrKeyInvalid) || xerrors.Is(err, jose.ErrCryptoFailure) || xerrors.Is(err, jwtutils.ErrMissingKeyID) { // These errors are probably because the user is mixing 2 coder deployments. return database.User{}, idpsync.HTTPError{ Code: http.StatusBadRequest, diff --git a/coderd/userauth_test.go b/coderd/userauth_test.go index 20dfe7f723899..6386be7eb8be4 100644 --- a/coderd/userauth_test.go +++ b/coderd/userauth_test.go @@ -3,6 +3,8 @@ package coderd_test import ( "context" "crypto" + "crypto/rand" + "encoding/json" "fmt" "io" "net/http" @@ -13,6 +15,7 @@ import ( "time" "github.com/coreos/go-oidc/v3/oidc" + "github.com/go-jose/go-jose/v4" "github.com/golang-jwt/jwt/v4" "github.com/google/go-github/v43/github" "github.com/google/uuid" @@ -27,10 +30,12 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/coderdtest/oidctest" + "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbgen" "github.com/coder/coder/v2/coderd/database/dbtestutil" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/notifications" "github.com/coder/coder/v2/coderd/promoauth" "github.com/coder/coder/v2/codersdk" @@ -1316,6 +1321,7 @@ func TestUserOIDC(t *testing.T) { owner := coderdtest.CreateFirstUser(t, client) user, userData := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID) + require.Equal(t, codersdk.LoginTypePassword, userData.LoginType) claims := jwt.MapClaims{ "email": userData.Email, @@ -1323,15 +1329,17 @@ func TestUserOIDC(t *testing.T) { var err error user.HTTPClient.Jar, err = cookiejar.New(nil) require.NoError(t, err) + user.HTTPClient.Transport = http.DefaultTransport.(*http.Transport).Clone() ctx := testutil.Context(t, testutil.WaitShort) + convertResponse, err := user.ConvertLoginType(ctx, codersdk.ConvertLoginRequest{ ToType: codersdk.LoginTypeOIDC, Password: "SomeSecurePassword!", }) require.NoError(t, err) - fake.LoginWithClient(t, user, claims, func(r *http.Request) { + _, _ = fake.LoginWithClient(t, user, claims, func(r *http.Request) { r.URL.RawQuery = url.Values{ "oidc_merge_state": {convertResponse.StateString}, }.Encode() @@ -1341,6 +1349,99 @@ func TestUserOIDC(t *testing.T) { r.AddCookie(cookie) } }) + + info, err := client.User(ctx, userData.ID.String()) + require.NoError(t, err) + require.Equal(t, codersdk.LoginTypeOIDC, info.LoginType) + }) + + t.Run("BadJWT", func(t *testing.T) { + t.Parallel() + + var ( + ctx = testutil.Context(t, testutil.WaitMedium) + logger = slogtest.Make(t, nil) + ) + + auditor := audit.NewMock() + fake := oidctest.NewFakeIDP(t, + oidctest.WithRefresh(func(_ string) error { + return xerrors.New("refreshing token should never occur") + }), + oidctest.WithServing(), + ) + cfg := fake.OIDCConfig(t, nil, func(cfg *coderd.OIDCConfig) { + cfg.AllowSignups = true + }) + + db, ps := dbtestutil.NewDB(t) + fetcher := &cryptokeys.DBFetcher{ + DB: db, + } + + kc, err := cryptokeys.NewSigningCache(ctx, logger, fetcher, codersdk.CryptoKeyFeatureOIDCConvert) + require.NoError(t, err) + + client := coderdtest.New(t, &coderdtest.Options{ + Auditor: auditor, + OIDCConfig: cfg, + Database: db, + Pubsub: ps, + OIDCConvertKeyCache: kc, + }) + + owner := coderdtest.CreateFirstUser(t, client) + user, userData := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID) + + claims := jwt.MapClaims{ + "email": userData.Email, + } + user.HTTPClient.Jar, err = cookiejar.New(nil) + require.NoError(t, err) + user.HTTPClient.Transport = http.DefaultTransport.(*http.Transport).Clone() + + convertResponse, err := user.ConvertLoginType(ctx, codersdk.ConvertLoginRequest{ + ToType: codersdk.LoginTypeOIDC, + Password: "SomeSecurePassword!", + }) + require.NoError(t, err) + + // Update the cookie to use a bad signing key. We're asserting the behavior of the scenario + // where a JWT gets minted on an old version of Coder but gets verified on a new version. + _, resp := fake.AttemptLogin(t, user, claims, func(r *http.Request) { + r.URL.RawQuery = url.Values{ + "oidc_merge_state": {convertResponse.StateString}, + }.Encode() + r.Header.Set(codersdk.SessionTokenHeader, user.SessionToken()) + + cookies := user.HTTPClient.Jar.Cookies(user.URL) + for i, cookie := range cookies { + if cookie.Name != coderd.OAuthConvertCookieValue { + continue + } + + jwt := cookie.Value + var claims coderd.OAuthConvertStateClaims + err := jwtutils.Verify(ctx, kc, jwt, &claims) + require.NoError(t, err) + badJWT := generateBadJWT(t, claims) + cookie.Value = badJWT + cookies[i] = cookie + } + + user.HTTPClient.Jar.SetCookies(user.URL, cookies) + + for _, cookie := range cookies { + fmt.Printf("cookie: %+v\n", cookie) + r.AddCookie(cookie) + } + }) + defer resp.Body.Close() + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + var respErr codersdk.Response + err = json.NewDecoder(resp.Body).Decode(&respErr) + require.NoError(t, err) + require.Contains(t, respErr.Message, "Using an invalid jwt to authorize this action.") }) t.Run("AlternateUsername", func(t *testing.T) { @@ -2022,3 +2123,24 @@ func inflateClaims(t testing.TB, seed jwt.MapClaims, size int) jwt.MapClaims { seed["random_data"] = junk return seed } + +// generateBadJWT generates a JWT with a random key. It's intended to emulate the old-style JWT's we generated. +func generateBadJWT(t *testing.T, claims interface{}) string { + t.Helper() + + var buf [64]byte + _, err := rand.Read(buf[:]) + require.NoError(t, err) + signer, err := jose.NewSigner(jose.SigningKey{ + Algorithm: jose.HS512, + Key: buf[:], + }, nil) + require.NoError(t, err) + payload, err := json.Marshal(claims) + require.NoError(t, err) + signed, err := signer.Sign(payload) + require.NoError(t, err) + compact, err := signed.CompactSerialize() + require.NoError(t, err) + return compact +} From 5ee6ad5a462dd938605c8d375422686ef9f00519 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 21 Oct 2024 22:59:02 +0000 Subject: [PATCH 30/39] add test for tailnet_resume jwt --- coderd/workspaceagents.go | 12 ++- coderd/workspaceagents_test.go | 189 ++++++++++++++++++++++----------- 2 files changed, 136 insertions(+), 65 deletions(-) diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 8767d2412e753..ce3ea61dde4ac 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -32,6 +32,7 @@ import ( "github.com/coder/coder/v2/coderd/externalauth" "github.com/coder/coder/v2/coderd/httpapi" "github.com/coder/coder/v2/coderd/httpmw" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/rbac/policy" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/agentsdk" @@ -854,7 +855,11 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R if resumeToken != "" { var err error peerID, err = api.Options.CoordinatorResumeTokenProvider.VerifyResumeToken(ctx, resumeToken) - if err != nil { + // If the token is missing the key ID, it's probably an old token in which + // case we just want to generate a new peer ID. + if xerrors.Is(err, jwtutils.ErrMissingKeyID) { + peerID = uuid.New() + } else if err != nil { httpapi.Write(ctx, rw, http.StatusUnauthorized, codersdk.Response{ Message: workspacesdk.CoordinateAPIInvalidResumeToken, Detail: err.Error(), @@ -863,9 +868,10 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R }, }) return + } else { + api.Logger.Debug(ctx, "accepted coordinate resume token for peer", + slog.F("peer_id", peerID.String())) } - api.Logger.Debug(ctx, "accepted coordinate resume token for peer", - slog.F("peer_id", peerID.String())) } api.WebsocketWaitMutex.Lock() diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 04aa5191901ec..9c2a93f8e9f07 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/go-jose/go-jose/v4/jwt" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,6 +38,7 @@ import ( "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/database/pubsub" "github.com/coder/coder/v2/coderd/externalauth" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/codersdk/agentsdk" "github.com/coder/coder/v2/codersdk/workspacesdk" @@ -555,73 +557,136 @@ func (r *resumeTokenRecordingProvider) VerifyResumeToken(ctx context.Context, to func TestWorkspaceAgentClientCoordinate_ResumeToken(t *testing.T) { t.Parallel() - logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) - clock := quartz.NewMock(t) - resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() - mgr := cryptokeys.StaticKey{ - ID: uuid.New().String(), - Key: resumeTokenSigningKey[:], - } - require.NoError(t, err) - resumeTokenProvider := newResumeTokenRecordingProvider( - t, - tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour), - ) - client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ - Coordinator: tailnet.NewCoordinator(logger), - CoordinatorResumeTokenProvider: resumeTokenProvider, + t.Run("OK", func(t *testing.T) { + t.Parallel() + + logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) + clock := quartz.NewMock(t) + resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() + mgr := cryptokeys.StaticKey{ + ID: uuid.New().String(), + Key: resumeTokenSigningKey[:], + } + require.NoError(t, err) + resumeTokenProvider := newResumeTokenRecordingProvider( + t, + tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour), + ) + client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Coordinator: tailnet.NewCoordinator(logger), + CoordinatorResumeTokenProvider: resumeTokenProvider, + }) + defer closer.Close() + user := coderdtest.CreateFirstUser(t, client) + + // Create a workspace with an agent. No need to connect it since clients can + // still connect to the coordinator while the agent isn't connected. + r := dbfake.WorkspaceBuild(t, api.Database, database.Workspace{ + OrganizationID: user.OrganizationID, + OwnerID: user.UserID, + }).WithAgent().Do() + agentTokenUUID, err := uuid.Parse(r.AgentToken) + require.NoError(t, err) + ctx := testutil.Context(t, testutil.WaitLong) + agentAndBuild, err := api.Database.GetWorkspaceAgentAndLatestBuildByAuthToken(dbauthz.AsSystemRestricted(ctx), agentTokenUUID) //nolint + require.NoError(t, err) + + // Connect with no resume token, and ensure that the peer ID is set to a + // random value. + originalResumeToken, err := connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, "") + require.NoError(t, err) + originalPeerID := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.generateCalls) + require.NotEqual(t, originalPeerID, uuid.Nil) + + // Connect with a valid resume token, and ensure that the peer ID is set to + // the stored value. + clock.Advance(time.Second) + newResumeToken, err := connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, originalResumeToken) + require.NoError(t, err) + verifiedToken := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.verifyCalls) + require.Equal(t, originalResumeToken, verifiedToken) + newPeerID := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.generateCalls) + require.Equal(t, originalPeerID, newPeerID) + require.NotEqual(t, originalResumeToken, newResumeToken) + + // Connect with an invalid resume token, and ensure that the request is + // rejected. + clock.Advance(time.Second) + _, err = connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, "invalid") + require.Error(t, err) + var sdkErr *codersdk.Error + require.ErrorAs(t, err, &sdkErr) + require.Equal(t, http.StatusUnauthorized, sdkErr.StatusCode()) + require.Len(t, sdkErr.Validations, 1) + require.Equal(t, "resume_token", sdkErr.Validations[0].Field) + verifiedToken = testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.verifyCalls) + require.Equal(t, "invalid", verifiedToken) + + select { + case <-resumeTokenProvider.generateCalls: + t.Fatal("unexpected peer ID in channel") + default: + } }) - defer closer.Close() - user := coderdtest.CreateFirstUser(t, client) - // Create a workspace with an agent. No need to connect it since clients can - // still connect to the coordinator while the agent isn't connected. - r := dbfake.WorkspaceBuild(t, api.Database, database.Workspace{ - OrganizationID: user.OrganizationID, - OwnerID: user.UserID, - }).WithAgent().Do() - agentTokenUUID, err := uuid.Parse(r.AgentToken) - require.NoError(t, err) - ctx := testutil.Context(t, testutil.WaitLong) - agentAndBuild, err := api.Database.GetWorkspaceAgentAndLatestBuildByAuthToken(dbauthz.AsSystemRestricted(ctx), agentTokenUUID) //nolint - require.NoError(t, err) + t.Run("BadJWT", func(t *testing.T) { + t.Parallel() - // Connect with no resume token, and ensure that the peer ID is set to a - // random value. - originalResumeToken, err := connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, "") - require.NoError(t, err) - originalPeerID := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.generateCalls) - require.NotEqual(t, originalPeerID, uuid.Nil) + logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) + clock := quartz.NewMock(t) + resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() + mgr := cryptokeys.StaticKey{ + ID: uuid.New().String(), + Key: resumeTokenSigningKey[:], + } + require.NoError(t, err) + resumeTokenProvider := newResumeTokenRecordingProvider( + t, + tailnet.NewResumeTokenKeyProvider(mgr, clock, time.Hour), + ) + client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{ + Coordinator: tailnet.NewCoordinator(logger), + CoordinatorResumeTokenProvider: resumeTokenProvider, + }) + defer closer.Close() + user := coderdtest.CreateFirstUser(t, client) - // Connect with a valid resume token, and ensure that the peer ID is set to - // the stored value. - clock.Advance(time.Second) - newResumeToken, err := connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, originalResumeToken) - require.NoError(t, err) - verifiedToken := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.verifyCalls) - require.Equal(t, originalResumeToken, verifiedToken) - newPeerID := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.generateCalls) - require.Equal(t, originalPeerID, newPeerID) - require.NotEqual(t, originalResumeToken, newResumeToken) - - // Connect with an invalid resume token, and ensure that the request is - // rejected. - clock.Advance(time.Second) - _, err = connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, "invalid") - require.Error(t, err) - var sdkErr *codersdk.Error - require.ErrorAs(t, err, &sdkErr) - require.Equal(t, http.StatusUnauthorized, sdkErr.StatusCode()) - require.Len(t, sdkErr.Validations, 1) - require.Equal(t, "resume_token", sdkErr.Validations[0].Field) - verifiedToken = testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.verifyCalls) - require.Equal(t, "invalid", verifiedToken) + // Create a workspace with an agent. No need to connect it since clients can + // still connect to the coordinator while the agent isn't connected. + r := dbfake.WorkspaceBuild(t, api.Database, database.Workspace{ + OrganizationID: user.OrganizationID, + OwnerID: user.UserID, + }).WithAgent().Do() + agentTokenUUID, err := uuid.Parse(r.AgentToken) + require.NoError(t, err) + ctx := testutil.Context(t, testutil.WaitLong) + agentAndBuild, err := api.Database.GetWorkspaceAgentAndLatestBuildByAuthToken(dbauthz.AsSystemRestricted(ctx), agentTokenUUID) //nolint + require.NoError(t, err) - select { - case <-resumeTokenProvider.generateCalls: - t.Fatal("unexpected peer ID in channel") - default: - } + // Connect with no resume token, and ensure that the peer ID is set to a + // random value. + originalResumeToken, err := connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, "") + require.NoError(t, err) + originalPeerID := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.generateCalls) + require.NotEqual(t, originalPeerID, uuid.Nil) + + // Connect with an outdated token, and ensure that the peer ID is set to a + // random value. We don't want to fail requests just because + // a user got unlucky during a deployment upgrade. + outdatedToken := generateBadJWT(t, jwtutils.RegisteredClaims{ + Subject: originalPeerID.String(), + Expiry: jwt.NewNumericDate(clock.Now().Add(time.Minute)), + }) + + clock.Advance(time.Second) + newResumeToken, err := connectToCoordinatorAndFetchResumeToken(ctx, logger, client, agentAndBuild.WorkspaceAgent.ID, outdatedToken) + require.NoError(t, err) + verifiedToken := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.verifyCalls) + require.Equal(t, outdatedToken, verifiedToken) + newPeerID := testutil.RequireRecvCtx(ctx, t, resumeTokenProvider.generateCalls) + require.NotEqual(t, originalPeerID, newPeerID) + require.NotEqual(t, originalResumeToken, newResumeToken) + }) } // connectToCoordinatorAndFetchResumeToken connects to the tailnet coordinator From 886d87c133ada41fc6c655ead05ae405c5bda02f Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Mon, 21 Oct 2024 23:58:34 +0000 Subject: [PATCH 31/39] add test for workspaceapps --- coderd/workspaceapps/apptest/apptest.go | 90 ++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/coderd/workspaceapps/apptest/apptest.go b/coderd/workspaceapps/apptest/apptest.go index 14adf2d61d362..f5f14cf00e3be 100644 --- a/coderd/workspaceapps/apptest/apptest.go +++ b/coderd/workspaceapps/apptest/apptest.go @@ -3,6 +3,7 @@ package apptest import ( "bufio" "context" + "crypto/rand" "encoding/json" "fmt" "io" @@ -463,7 +464,7 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { appClient.SetSessionToken("") // Try to load the application without authentication. - u := c.appURL + u := *c.appURL u.Path = path.Join(u.Path, "/test") req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) require.NoError(t, err) @@ -500,7 +501,7 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { // Copy the query parameters and then check equality. u.RawQuery = gotLocation.RawQuery - require.Equal(t, u, gotLocation) + require.Equal(t, u, *gotLocation) // Verify the API key is set. encryptedAPIKey := gotLocation.Query().Get(workspaceapps.SubdomainProxyAPIKeyParam) @@ -580,6 +581,70 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { resp.Body.Close() require.Equal(t, http.StatusOK, resp.StatusCode) }) + + t.Run("BadJWT", func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + currentKeyStr := appDetails.SDKClient.SessionToken() + appClient := appDetails.AppClient(t) + appClient.SetSessionToken("") + u := *c.appURL + u.Path = path.Join(u.Path, "/test") + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + require.NoError(t, err) + + var resp *http.Response + resp, err = doWithRetries(t, appClient, req) + require.NoError(t, err) + + if !assert.Equal(t, http.StatusSeeOther, resp.StatusCode) { + dump, err := httputil.DumpResponse(resp, true) + require.NoError(t, err) + t.Log(string(dump)) + } + resp.Body.Close() + + // Check that the Location is correct. + gotLocation, err := resp.Location() + require.NoError(t, err) + // This should always redirect to the primary access URL. + require.Equal(t, appDetails.SDKClient.URL.Host, gotLocation.Host) + require.Equal(t, "/api/v2/applications/auth-redirect", gotLocation.Path) + require.Equal(t, u.String(), gotLocation.Query().Get("redirect_uri")) + + // Load the application auth-redirect endpoint. + resp, err = requestWithRetries(ctx, t, appDetails.SDKClient, http.MethodGet, "/api/v2/applications/auth-redirect", nil, codersdk.WithQueryParam( + "redirect_uri", u.String(), + )) + require.NoError(t, err) + defer resp.Body.Close() + + require.Equal(t, http.StatusSeeOther, resp.StatusCode) + gotLocation, err = resp.Location() + require.NoError(t, err) + + badToken := generateBadJWT(t, workspaceapps.EncryptedAPIKeyPayload{ + APIKey: currentKeyStr, + }) + + gotLocation.RawQuery = (url.Values{ + workspaceapps.SubdomainProxyAPIKeyParam: {badToken}, + }).Encode() + + req, err = http.NewRequestWithContext(ctx, "GET", gotLocation.String(), nil) + require.NoError(t, err) + resp, err = appClient.HTTPClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusBadRequest, resp.StatusCode) + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Contains(t, string(body), "Could not decrypt API key. Please remove the query parameter and try again.") + + }) } }) }) @@ -1789,3 +1854,24 @@ func assertWorkspaceLastUsedAtNotUpdated(t testing.TB, details *Details) { require.NoError(t, err) require.Equal(t, before.LastUsedAt, after.LastUsedAt, "workspace LastUsedAt updated when it should not have been") } + +// generateBadJWT generates a JWT with a random key. It's intended to emulate the old-style JWT's we generated. +func generateBadJWT(t *testing.T, claims interface{}) string { + t.Helper() + + var buf [64]byte + _, err := rand.Read(buf[:]) + require.NoError(t, err) + signer, err := jose.NewSigner(jose.SigningKey{ + Algorithm: jose.HS512, + Key: buf[:], + }, nil) + require.NoError(t, err) + payload, err := json.Marshal(claims) + require.NoError(t, err) + signed, err := signer.Sign(payload) + require.NoError(t, err) + compact, err := signed.CompactSerialize() + require.NoError(t, err) + return compact +} From 526144283323eb1cc08fca2371057a253c2e09be Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 22 Oct 2024 00:35:09 +0000 Subject: [PATCH 32/39] add test for signedtoken --- coderd/workspaceapps/apptest/apptest.go | 160 +++++++++++++++++++++++- 1 file changed, 158 insertions(+), 2 deletions(-) diff --git a/coderd/workspaceapps/apptest/apptest.go b/coderd/workspaceapps/apptest/apptest.go index f5f14cf00e3be..69a66de2d373b 100644 --- a/coderd/workspaceapps/apptest/apptest.go +++ b/coderd/workspaceapps/apptest/apptest.go @@ -409,6 +409,67 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { require.Equal(t, http.StatusInternalServerError, resp.StatusCode) assertWorkspaceLastUsedAtNotUpdated(t, appDetails) }) + + t.Run("BadJWT", func(t *testing.T) { + t.Parallel() + + appDetails := setupProxyTest(t, nil) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + u := appDetails.PathAppURL(appDetails.Apps.Owner) + resp, err := requestWithRetries(ctx, t, appDetails.AppClient(t), http.MethodGet, u.String(), nil) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, proxyTestAppBody, string(body)) + require.Equal(t, http.StatusOK, resp.StatusCode) + + appTokenCookie := findCookie(resp.Cookies(), codersdk.SignedAppTokenCookie) + require.NotNil(t, appTokenCookie, "no signed app token cookie in response") + require.Equal(t, appTokenCookie.Path, u.Path, "incorrect path on app token cookie") + + object, err := jose.ParseSigned(appTokenCookie.Value) + require.NoError(t, err) + require.Len(t, object.Signatures, 1) + + // Parse the payload. + var tok workspaceapps.SignedToken + //nolint:gosec + err = json.Unmarshal(object.UnsafePayloadWithoutVerification(), &tok) + require.NoError(t, err) + + appTokenClient := appDetails.AppClient(t) + apiKey := appTokenClient.SessionToken() + appTokenClient.SetSessionToken("") + appTokenClient.HTTPClient.Jar, err = cookiejar.New(nil) + require.NoError(t, err) + // Sign the token with an old-style key. + appTokenCookie.Value = generateBadJWT(t, tok) + appTokenClient.HTTPClient.Jar.SetCookies(u, + []*http.Cookie{ + appTokenCookie, + { + Name: codersdk.PathAppSessionTokenCookie, + Value: apiKey, + }, + }, + ) + + resp, err = requestWithRetries(ctx, t, appTokenClient, http.MethodGet, u.String(), nil) + require.NoError(t, err) + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, proxyTestAppBody, string(body)) + require.Equal(t, http.StatusOK, resp.StatusCode) + assertWorkspaceLastUsedAtUpdated(t, appDetails) + + // Since the old token is invalid, the signed app token cookie should have a new value. + newTokenCookie := findCookie(resp.Cookies(), codersdk.SignedAppTokenCookie) + require.NotEqual(t, appTokenCookie.Value, newTokenCookie.Value) + }) }) t.Run("WorkspaceApplicationAuth", func(t *testing.T) { @@ -582,7 +643,7 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { require.Equal(t, http.StatusOK, resp.StatusCode) }) - t.Run("BadJWT", func(t *testing.T) { + t.Run("BadJWE", func(t *testing.T) { t.Parallel() ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) @@ -626,7 +687,7 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { gotLocation, err = resp.Location() require.NoError(t, err) - badToken := generateBadJWT(t, workspaceapps.EncryptedAPIKeyPayload{ + badToken := generateBadJWE(t, workspaceapps.EncryptedAPIKeyPayload{ APIKey: currentKeyStr, }) @@ -1142,6 +1203,68 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { assertWorkspaceLastUsedAtNotUpdated(t, appDetails) }) }) + + t.Run("BadJWT", func(t *testing.T) { + t.Parallel() + + appDetails := setupProxyTest(t, nil) + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + u := appDetails.SubdomainAppURL(appDetails.Apps.Owner) + resp, err := requestWithRetries(ctx, t, appDetails.AppClient(t), http.MethodGet, u.String(), nil) + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, proxyTestAppBody, string(body)) + require.Equal(t, http.StatusOK, resp.StatusCode) + + appTokenCookie := findCookie(resp.Cookies(), codersdk.SignedAppTokenCookie) + require.NotNil(t, appTokenCookie, "no signed token cookie in response") + require.Equal(t, appTokenCookie.Path, "/", "incorrect path on signed token cookie") + + object, err := jose.ParseSigned(appTokenCookie.Value) + require.NoError(t, err) + require.Len(t, object.Signatures, 1) + + // Parse the payload. + var tok workspaceapps.SignedToken + //nolint:gosec + err = json.Unmarshal(object.UnsafePayloadWithoutVerification(), &tok) + require.NoError(t, err) + + appTokenClient := appDetails.AppClient(t) + apiKey := appTokenClient.SessionToken() + appTokenClient.SetSessionToken("") + appTokenClient.HTTPClient.Jar, err = cookiejar.New(nil) + require.NoError(t, err) + // Sign the token with an old-style key. + appTokenCookie.Value = generateBadJWT(t, tok) + appTokenClient.HTTPClient.Jar.SetCookies(u, + []*http.Cookie{ + appTokenCookie, + { + Name: codersdk.SubdomainAppSessionTokenCookie, + Value: apiKey, + }, + }, + ) + + // We should still be able to successfully proxy. + resp, err = requestWithRetries(ctx, t, appTokenClient, http.MethodGet, u.String(), nil) + require.NoError(t, err) + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, proxyTestAppBody, string(body)) + require.Equal(t, http.StatusOK, resp.StatusCode) + assertWorkspaceLastUsedAtUpdated(t, appDetails) + + // Since the old token is invalid, the signed app token cookie should have a new value. + newTokenCookie := findCookie(resp.Cookies(), codersdk.SignedAppTokenCookie) + require.NotEqual(t, appTokenCookie.Value, newTokenCookie.Value) + }) }) t.Run("PortSharing", func(t *testing.T) { @@ -1855,6 +1978,30 @@ func assertWorkspaceLastUsedAtNotUpdated(t testing.TB, details *Details) { require.Equal(t, before.LastUsedAt, after.LastUsedAt, "workspace LastUsedAt updated when it should not have been") } +func generateBadJWE(t *testing.T, claims interface{}) string { + t.Helper() + var buf [32]byte + _, err := rand.Read(buf[:]) + require.NoError(t, err) + encrypt, err := jose.NewEncrypter( + jose.A256GCM, + jose.Recipient{ + Algorithm: jose.A256GCMKW, + Key: buf[:], + }, &jose.EncrypterOptions{ + Compression: jose.DEFLATE, + }, + ) + require.NoError(t, err) + payload, err := json.Marshal(claims) + require.NoError(t, err) + signed, err := encrypt.Encrypt(payload) + require.NoError(t, err) + compact, err := signed.CompactSerialize() + require.NoError(t, err) + return compact +} + // generateBadJWT generates a JWT with a random key. It's intended to emulate the old-style JWT's we generated. func generateBadJWT(t *testing.T, claims interface{}) string { t.Helper() @@ -1875,3 +2022,12 @@ func generateBadJWT(t *testing.T, claims interface{}) string { require.NoError(t, err) return compact } + +func findCookie(cookies []*http.Cookie, name string) *http.Cookie { + for _, cookie := range cookies { + if cookie.Name == name { + return cookie + } + } + return nil +} From 4a1d97491c1719a640a4beb04a8ba0de9975bbdb Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 22 Oct 2024 00:43:42 +0000 Subject: [PATCH 33/39] fix migrations --- ...tokey_features.down.sql => 000268_cryptokey_features.down.sql} | 0 ...cryptokey_features.up.sql => 000268_cryptokey_features.up.sql} | 0 ...cryptokey_features.up.sql => 000268_cryptokey_features.up.sql} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000267_cryptokey_features.down.sql => 000268_cryptokey_features.down.sql} (100%) rename coderd/database/migrations/{000267_cryptokey_features.up.sql => 000268_cryptokey_features.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000267_cryptokey_features.up.sql => 000268_cryptokey_features.up.sql} (100%) diff --git a/coderd/database/migrations/000267_cryptokey_features.down.sql b/coderd/database/migrations/000268_cryptokey_features.down.sql similarity index 100% rename from coderd/database/migrations/000267_cryptokey_features.down.sql rename to coderd/database/migrations/000268_cryptokey_features.down.sql diff --git a/coderd/database/migrations/000267_cryptokey_features.up.sql b/coderd/database/migrations/000268_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/000267_cryptokey_features.up.sql rename to coderd/database/migrations/000268_cryptokey_features.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000267_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000268_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000267_cryptokey_features.up.sql rename to coderd/database/migrations/testdata/fixtures/000268_cryptokey_features.up.sql From 092a24183fb5193bd87503993ea2039ec485d15e Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 22 Oct 2024 00:43:58 +0000 Subject: [PATCH 34/39] fmt --- coderd/workspaceapps/apptest/apptest.go | 1 - 1 file changed, 1 deletion(-) diff --git a/coderd/workspaceapps/apptest/apptest.go b/coderd/workspaceapps/apptest/apptest.go index 69a66de2d373b..580336703113a 100644 --- a/coderd/workspaceapps/apptest/apptest.go +++ b/coderd/workspaceapps/apptest/apptest.go @@ -704,7 +704,6 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { body, err := io.ReadAll(resp.Body) require.NoError(t, err) require.Contains(t, string(body), "Could not decrypt API key. Please remove the query parameter and try again.") - }) } }) From 87828a2f04476845eeea6c7b70b17d2aa09525d6 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 22 Oct 2024 15:11:08 +0000 Subject: [PATCH 35/39] pr comments --- coderd/coderd.go | 13 ++------ coderd/cryptokeys/cache.go | 11 +++---- coderd/cryptokeys/rotate.go | 6 ++-- coderd/cryptokeys/rotate_test.go | 6 ++-- coderd/workspaceapps/apptest/apptest.go | 41 +++---------------------- 5 files changed, 16 insertions(+), 61 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index ae23d15d6b239..3011c2d58d39c 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -492,10 +492,7 @@ func New(options *Options) *API { // Start a background process that rotates keys. We intentionally start this after the caches // are created to force initial requests for a key to populate the caches. This helps catch // bugs that may only occur when a key isn't precached in tests and the latency cost is minimal. - err = cryptokeys.StartRotator(ctx, options.Logger, options.Database) - if err != nil { - must(options.Logger, "failed to start key rotator", err) - } + cryptokeys.StartRotator(ctx, options.Logger, options.Database) api := &API{ ctx: ctx, @@ -659,7 +656,7 @@ func New(options *Options) *API { ResumeTokenProvider: api.Options.CoordinatorResumeTokenProvider, }) if err != nil { - must(api.Logger, "failed to initialize tailnet client service", err) + api.Logger.Fatal(context.Background(), "failed to initialize tailnet client service", slog.Error(err)) } api.statsReporter = workspacestats.NewReporter(workspacestats.ReporterOptions{ @@ -1657,9 +1654,3 @@ func ReadExperiments(log slog.Logger, raw []string) codersdk.Experiments { } return exps } - -func must(logger slog.Logger, msg string, err error) { - if err != nil { - logger.Fatal(context.Background(), msg, slog.Error(err)) - } -} diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index e4ab9cde7e625..dfb2061b98ecc 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -110,7 +110,7 @@ func NewSigningCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, return nil, xerrors.Errorf("invalid feature: %s", feature) } logger = logger.Named(fmt.Sprintf("%s_signing_keycache", feature)) - return newCache(ctx, logger, fetcher, feature, opts...) + return newCache(ctx, logger, fetcher, feature, opts...), nil } func NewEncryptionCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, @@ -120,10 +120,10 @@ func NewEncryptionCache(ctx context.Context, logger slog.Logger, fetcher Fetcher return nil, xerrors.Errorf("invalid feature: %s", feature) } logger = logger.Named(fmt.Sprintf("%s_encryption_keycache", feature)) - return newCache(ctx, logger, fetcher, feature, opts...) + return newCache(ctx, logger, fetcher, feature, opts...), nil } -func newCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, feature codersdk.CryptoKeyFeature, opts ...func(*cache)) (*cache, error) { +func newCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, feature codersdk.CryptoKeyFeature, opts ...func(*cache)) *cache { cache := &cache{ clock: quartz.NewReal(), logger: logger, @@ -142,11 +142,10 @@ func newCache(ctx context.Context, logger slog.Logger, fetcher Fetcher, feature keys, err := cache.cryptoKeys(cache.ctx) if err != nil { - cache.cancel() - return nil, xerrors.Errorf("initial fetch: %w", err) + cache.logger.Critical(cache.ctx, "failed initial fetch", slog.Error(err)) } cache.keys = keys - return cache, nil + return cache } func (c *cache) EncryptingKey(ctx context.Context) (string, interface{}, error) { diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index 090100fd8bf32..5d7d7b33b9dec 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -54,7 +54,7 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption { // StartRotator starts a background process that rotates keys in the database. // It ensures there's at least one valid key per feature prior to returning. // Canceling the provided context will stop the background process. -func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, opts ...RotatorOption) error { +func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, opts ...RotatorOption) { //nolint:gocritic // KeyRotator can only rotate crypto keys. ctx = dbauthz.AsKeyRotator(ctx) kr := &rotator{ @@ -71,12 +71,10 @@ func StartRotator(ctx context.Context, logger slog.Logger, db database.Store, op err := kr.rotateKeys(ctx) if err != nil { - return xerrors.Errorf("rotate keys: %w", err) + kr.logger.Critical(ctx, "failed to rotate keys", slog.Error(err)) } go kr.start(ctx) - - return nil } // start begins the process of rotating keys. diff --git a/coderd/cryptokeys/rotate_test.go b/coderd/cryptokeys/rotate_test.go index 96acacf1d0182..9e147c8f921f0 100644 --- a/coderd/cryptokeys/rotate_test.go +++ b/coderd/cryptokeys/rotate_test.go @@ -34,8 +34,7 @@ func TestRotator(t *testing.T) { require.NoError(t, err) require.Len(t, dbkeys, 0) - err = cryptokeys.StartRotator(ctx, logger, db, cryptokeys.WithClock(clock)) - require.NoError(t, err) + cryptokeys.StartRotator(ctx, logger, db, cryptokeys.WithClock(clock)) // Fetch the keys from the database and ensure they // are as expected. @@ -66,8 +65,7 @@ func TestRotator(t *testing.T) { trap := clock.Trap().TickerFunc() t.Cleanup(trap.Close) - err := cryptokeys.StartRotator(ctx, logger, db, cryptokeys.WithClock(clock)) - require.NoError(t, err) + cryptokeys.StartRotator(ctx, logger, db, cryptokeys.WithClock(clock)) initialKeyLen := len(database.AllCryptoKeyFeatureValues()) // Fetch the keys from the database and ensure they diff --git a/coderd/workspaceapps/apptest/apptest.go b/coderd/workspaceapps/apptest/apptest.go index 580336703113a..c6e251806230d 100644 --- a/coderd/workspaceapps/apptest/apptest.go +++ b/coderd/workspaceapps/apptest/apptest.go @@ -654,50 +654,19 @@ func Run(t *testing.T, appHostIsPrimary bool, factory DeploymentFactory) { appClient.SetSessionToken("") u := *c.appURL u.Path = path.Join(u.Path, "/test") - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) - require.NoError(t, err) - - var resp *http.Response - resp, err = doWithRetries(t, appClient, req) - require.NoError(t, err) - - if !assert.Equal(t, http.StatusSeeOther, resp.StatusCode) { - dump, err := httputil.DumpResponse(resp, true) - require.NoError(t, err) - t.Log(string(dump)) - } - resp.Body.Close() - - // Check that the Location is correct. - gotLocation, err := resp.Location() - require.NoError(t, err) - // This should always redirect to the primary access URL. - require.Equal(t, appDetails.SDKClient.URL.Host, gotLocation.Host) - require.Equal(t, "/api/v2/applications/auth-redirect", gotLocation.Path) - require.Equal(t, u.String(), gotLocation.Query().Get("redirect_uri")) - - // Load the application auth-redirect endpoint. - resp, err = requestWithRetries(ctx, t, appDetails.SDKClient, http.MethodGet, "/api/v2/applications/auth-redirect", nil, codersdk.WithQueryParam( - "redirect_uri", u.String(), - )) - require.NoError(t, err) - defer resp.Body.Close() - - require.Equal(t, http.StatusSeeOther, resp.StatusCode) - gotLocation, err = resp.Location() - require.NoError(t, err) - badToken := generateBadJWE(t, workspaceapps.EncryptedAPIKeyPayload{ APIKey: currentKeyStr, }) - gotLocation.RawQuery = (url.Values{ + u.RawQuery = (url.Values{ workspaceapps.SubdomainProxyAPIKeyParam: {badToken}, }).Encode() - req, err = http.NewRequestWithContext(ctx, "GET", gotLocation.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) require.NoError(t, err) - resp, err = appClient.HTTPClient.Do(req) + + var resp *http.Response + resp, err = doWithRetries(t, appClient, req) require.NoError(t, err) defer resp.Body.Close() require.Equal(t, http.StatusBadRequest, resp.StatusCode) From 358aaa856c68c92262d1701abcdc23548ea68594 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Tue, 22 Oct 2024 15:21:24 +0000 Subject: [PATCH 36/39] Rename cryptokey migration files to update sequence --- ...tokey_features.down.sql => 000270_cryptokey_features.down.sql} | 0 ...cryptokey_features.up.sql => 000270_cryptokey_features.up.sql} | 0 ...cryptokey_features.up.sql => 000270_cryptokey_features.up.sql} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000268_cryptokey_features.down.sql => 000270_cryptokey_features.down.sql} (100%) rename coderd/database/migrations/{000268_cryptokey_features.up.sql => 000270_cryptokey_features.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000268_cryptokey_features.up.sql => 000270_cryptokey_features.up.sql} (100%) diff --git a/coderd/database/migrations/000268_cryptokey_features.down.sql b/coderd/database/migrations/000270_cryptokey_features.down.sql similarity index 100% rename from coderd/database/migrations/000268_cryptokey_features.down.sql rename to coderd/database/migrations/000270_cryptokey_features.down.sql diff --git a/coderd/database/migrations/000268_cryptokey_features.up.sql b/coderd/database/migrations/000270_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/000268_cryptokey_features.up.sql rename to coderd/database/migrations/000270_cryptokey_features.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000268_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000270_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000268_cryptokey_features.up.sql rename to coderd/database/migrations/testdata/fixtures/000270_cryptokey_features.up.sql From 5798a332fd6d5fc510f73a5118d1a347015f2e75 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 24 Oct 2024 16:58:46 +0000 Subject: [PATCH 37/39] migrations --- ...tokey_features.down.sql => 000271_cryptokey_features.down.sql} | 0 ...cryptokey_features.up.sql => 000271_cryptokey_features.up.sql} | 0 ...cryptokey_features.up.sql => 000271_cryptokey_features.up.sql} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000270_cryptokey_features.down.sql => 000271_cryptokey_features.down.sql} (100%) rename coderd/database/migrations/{000270_cryptokey_features.up.sql => 000271_cryptokey_features.up.sql} (100%) rename coderd/database/migrations/testdata/fixtures/{000270_cryptokey_features.up.sql => 000271_cryptokey_features.up.sql} (100%) diff --git a/coderd/database/migrations/000270_cryptokey_features.down.sql b/coderd/database/migrations/000271_cryptokey_features.down.sql similarity index 100% rename from coderd/database/migrations/000270_cryptokey_features.down.sql rename to coderd/database/migrations/000271_cryptokey_features.down.sql diff --git a/coderd/database/migrations/000270_cryptokey_features.up.sql b/coderd/database/migrations/000271_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/000270_cryptokey_features.up.sql rename to coderd/database/migrations/000271_cryptokey_features.up.sql diff --git a/coderd/database/migrations/testdata/fixtures/000270_cryptokey_features.up.sql b/coderd/database/migrations/testdata/fixtures/000271_cryptokey_features.up.sql similarity index 100% rename from coderd/database/migrations/testdata/fixtures/000270_cryptokey_features.up.sql rename to coderd/database/migrations/testdata/fixtures/000271_cryptokey_features.up.sql From 200cd68453d56cbebc93b346fd291cf275369d16 Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 24 Oct 2024 17:41:40 +0000 Subject: [PATCH 38/39] Refactor StaticKey to jwtutils package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移動StaticKeyからcryptokeysパッケージへのjwtutils。 これにより、jwtutilsとのセキュリティおよび独立性の一貫性が強化されます。 また、tailnetの依存関係を減らします。 --- coderd/cryptokeys/cache.go | 32 -------------------------------- coderd/jwtutils/jws.go | 37 +++++++++++++++++++++++++++++++++++++ tailnet/resume.go | 7 +++---- 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/coderd/cryptokeys/cache.go b/coderd/cryptokeys/cache.go index dfb2061b98ecc..7777d5f75b942 100644 --- a/coderd/cryptokeys/cache.go +++ b/coderd/cryptokeys/cache.go @@ -379,38 +379,6 @@ func (c *cache) Close() error { return nil } -// StaticKey fulfills the SigningKeycache and EncryptionKeycache interfaces. Useful for testing. -type StaticKey struct { - ID string - Key interface{} -} - -func (s StaticKey) SigningKey(_ context.Context) (string, interface{}, error) { - return s.ID, s.Key, nil -} - -func (s StaticKey) VerifyingKey(_ context.Context, id string) (interface{}, error) { - if id != s.ID { - return nil, xerrors.Errorf("invalid id %q", id) - } - return s.Key, nil -} - -func (s StaticKey) EncryptingKey(_ context.Context) (string, interface{}, error) { - return s.ID, s.Key, nil -} - -func (s StaticKey) DecryptingKey(_ context.Context, id string) (interface{}, error) { - if id != s.ID { - return nil, xerrors.Errorf("invalid id %q", id) - } - return s.Key, nil -} - -func (StaticKey) Close() error { - return nil -} - // We have to do this to avoid a circular dependency on db2sdk (cryptokeys -> db2sdk -> tailnet -> cryptokeys) func toSDKKeys(keys []database.CryptoKey) []codersdk.CryptoKey { into := make([]codersdk.CryptoKey, 0, len(keys)) diff --git a/coderd/jwtutils/jws.go b/coderd/jwtutils/jws.go index f6a9964924424..0c8ca9aa30f39 100644 --- a/coderd/jwtutils/jws.go +++ b/coderd/jwtutils/jws.go @@ -41,6 +41,11 @@ const ( signingAlgo = jose.HS512 ) +type SigningKeyManager interface { + SigningKeyProvider + VerifyKeyProvider +} + type SigningKeyProvider interface { SigningKey(ctx context.Context) (id string, key interface{}, err error) } @@ -148,3 +153,35 @@ func Verify(ctx context.Context, v VerifyKeyProvider, token string, claims Claim return claims.Validate(options.RegisteredClaims) } + +// StaticKey fulfills the SigningKeycache and EncryptionKeycache interfaces. Useful for testing. +type StaticKey struct { + ID string + Key interface{} +} + +func (s StaticKey) SigningKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKey) VerifyingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} + +func (s StaticKey) EncryptingKey(_ context.Context) (string, interface{}, error) { + return s.ID, s.Key, nil +} + +func (s StaticKey) DecryptingKey(_ context.Context, id string) (interface{}, error) { + if id != s.ID { + return nil, xerrors.Errorf("invalid id %q", id) + } + return s.Key, nil +} + +func (StaticKey) Close() error { + return nil +} diff --git a/tailnet/resume.go b/tailnet/resume.go index 0201553f10316..2975fa35f1674 100644 --- a/tailnet/resume.go +++ b/tailnet/resume.go @@ -11,7 +11,6 @@ import ( "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" - "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/tailnet/proto" "github.com/coder/quartz" @@ -29,7 +28,7 @@ func NewInsecureTestResumeTokenProvider() ResumeTokenProvider { if err != nil { panic(err) } - return NewResumeTokenKeyProvider(cryptokeys.StaticKey{ + return NewResumeTokenKeyProvider(jwtutils.StaticKey{ ID: uuid.New().String(), Key: key[:], }, quartz.NewReal(), time.Hour) @@ -52,12 +51,12 @@ func GenerateResumeTokenSigningKey() (ResumeTokenSigningKey, error) { } type ResumeTokenKeyProvider struct { - key cryptokeys.SigningKeycache + key jwtutils.SigningKeyManager clock quartz.Clock expiry time.Duration } -func NewResumeTokenKeyProvider(key cryptokeys.SigningKeycache, clock quartz.Clock, expiry time.Duration) ResumeTokenProvider { +func NewResumeTokenKeyProvider(key jwtutils.SigningKeyManager, clock quartz.Clock, expiry time.Duration) ResumeTokenProvider { if expiry <= 0 { expiry = DefaultResumeTokenExpiry } From 2194b4d056675fbf452d6aab238fbf406876fe7c Mon Sep 17 00:00:00 2001 From: Jon Ayers Date: Thu, 24 Oct 2024 18:48:31 +0000 Subject: [PATCH 39/39] fix tests --- coderd/workspaceagents_test.go | 5 ++--- coderd/workspaceapps/token_test.go | 5 ++--- codersdk/workspacesdk/connector_internal_test.go | 6 +++--- tailnet/resume_test.go | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/coderd/workspaceagents_test.go b/coderd/workspaceagents_test.go index 631703b2bc273..ba677975471d6 100644 --- a/coderd/workspaceagents_test.go +++ b/coderd/workspaceagents_test.go @@ -29,7 +29,6 @@ import ( agentproto "github.com/coder/coder/v2/agent/proto" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/coderdtest/oidctest" - "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" "github.com/coder/coder/v2/coderd/database/dbfake" @@ -563,7 +562,7 @@ func TestWorkspaceAgentClientCoordinate_ResumeToken(t *testing.T) { logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() - mgr := cryptokeys.StaticKey{ + mgr := jwtutils.StaticKey{ ID: uuid.New().String(), Key: resumeTokenSigningKey[:], } @@ -635,7 +634,7 @@ func TestWorkspaceAgentClientCoordinate_ResumeToken(t *testing.T) { logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug) clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() - mgr := cryptokeys.StaticKey{ + mgr := jwtutils.StaticKey{ ID: uuid.New().String(), Key: resumeTokenSigningKey[:], } diff --git a/coderd/workspaceapps/token_test.go b/coderd/workspaceapps/token_test.go index 91f2ef69aa6b2..db070268fa196 100644 --- a/coderd/workspaceapps/token_test.go +++ b/coderd/workspaceapps/token_test.go @@ -15,7 +15,6 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" - "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/coderd/workspaceapps" ) @@ -344,10 +343,10 @@ func Test_FromRequest(t *testing.T) { }) } -func newSigner(t *testing.T) cryptokeys.StaticKey { +func newSigner(t *testing.T) jwtutils.StaticKey { t.Helper() - return cryptokeys.StaticKey{ + return jwtutils.StaticKey{ ID: "test", Key: generateSecret(t, 64), } diff --git a/codersdk/workspacesdk/connector_internal_test.go b/codersdk/workspacesdk/connector_internal_test.go index 1443f93cf7a55..19f1930c89bc5 100644 --- a/codersdk/workspacesdk/connector_internal_test.go +++ b/codersdk/workspacesdk/connector_internal_test.go @@ -24,8 +24,8 @@ import ( "cdr.dev/slog" "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/apiversion" - "github.com/coder/coder/v2/coderd/cryptokeys" "github.com/coder/coder/v2/coderd/httpapi" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/tailnet/proto" @@ -166,7 +166,7 @@ func TestTailnetAPIConnector_ResumeToken(t *testing.T) { clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - mgr := cryptokeys.StaticKey{ + mgr := jwtutils.StaticKey{ ID: "123", Key: resumeTokenSigningKey[:], } @@ -285,7 +285,7 @@ func TestTailnetAPIConnector_ResumeTokenFailure(t *testing.T) { clock := quartz.NewMock(t) resumeTokenSigningKey, err := tailnet.GenerateResumeTokenSigningKey() require.NoError(t, err) - mgr := cryptokeys.StaticKey{ + mgr := jwtutils.StaticKey{ ID: uuid.New().String(), Key: resumeTokenSigningKey[:], } diff --git a/tailnet/resume_test.go b/tailnet/resume_test.go index 80f2fc175dc22..6f32fba4c511e 100644 --- a/tailnet/resume_test.go +++ b/tailnet/resume_test.go @@ -9,7 +9,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" - "github.com/coder/coder/v2/coderd/cryptokeys" + "github.com/coder/coder/v2/coderd/jwtutils" "github.com/coder/coder/v2/tailnet" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" @@ -91,8 +91,8 @@ func TestResumeTokenKeyProvider(t *testing.T) { }) } -func newKeySigner(key tailnet.ResumeTokenSigningKey) cryptokeys.StaticKey { - return cryptokeys.StaticKey{ +func newKeySigner(key tailnet.ResumeTokenSigningKey) jwtutils.StaticKey { + return jwtutils.StaticKey{ ID: "123", Key: key[:], }