Skip to content

chore: add workspace proxies to the backend #7032

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 51 commits into from
Apr 17, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
ca5b50c
feat: Implement start of external workspace proxies
Emyrk Apr 5, 2023
5fc7832
Add more init code
Emyrk Apr 5, 2023
391fe74
feat: add proxysdk and proxy tokeng
deansheather Apr 6, 2023
23d0a4c
Comments and import cleanup
Emyrk Apr 6, 2023
7cce9a2
Move to wsproxy, make unit test work, update audit log resources
Emyrk Apr 6, 2023
2aebe77
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 6, 2023
020b4b5
Add proxy token provider
Emyrk Apr 6, 2023
dc5af55
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 6, 2023
c5225ae
Begin writing unit test for external proxy
Emyrk Apr 6, 2023
d6a1217
Add option validation
Emyrk Apr 7, 2023
1e163d9
Fix access url passing
Emyrk Apr 7, 2023
e86a518
Healthz and buildinfo endpoints
Emyrk Apr 7, 2023
20b44c6
do stuff
deansheather Apr 11, 2023
68c3bb1
Linting
Emyrk Apr 11, 2023
ec04552
Check workspace proxy hostnames for subdomain apps
Emyrk Apr 11, 2023
07323e5
Path based redirects redirect to dashboardurl
Emyrk Apr 11, 2023
ffa8b00
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 11, 2023
a96a73b
Just commit something
Emyrk Apr 11, 2023
2d7e242
use query instead of proxycache
deansheather Apr 12, 2023
e80e7e0
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 12, 2023
be25c51
Make gen
Emyrk Apr 12, 2023
208eaf1
MAke gen
Emyrk Apr 12, 2023
6cfb62c
Linting
Emyrk Apr 12, 2023
a112e29
Bump migration
Emyrk Apr 13, 2023
d6edd29
Smuggling for path apps on proxies
deansheather Apr 13, 2023
5db3d25
Reuse system rbac subject
Emyrk Apr 13, 2023
7e4ed87
Add TODO
Emyrk Apr 13, 2023
7140420
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 13, 2023
fb30e1a
Give moons exec perms
Emyrk Apr 13, 2023
22aadf1
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 13, 2023
a66ffd7
Merge remote-tracking branch 'origin/main' into dreamteam/external_proxy
Emyrk Apr 14, 2023
a483f3e
Fix merge mistake
Emyrk Apr 14, 2023
50fa1ca
Renames from PR feedback
Emyrk Apr 14, 2023
b7f3b86
Update enterprise/audit/table.go
Emyrk Apr 17, 2023
bb032c3
Renames and formatting
Emyrk Apr 17, 2023
6ab0dea
Make gen
Emyrk Apr 17, 2023
12c6f8d
Fix compile
Emyrk Apr 17, 2023
224fa2f
Add comments to sql columns
Emyrk Apr 17, 2023
06fb88b
ExternalProxy -> WorkspaceProxy
Emyrk Apr 17, 2023
82d10d9
Remove Actor function
Emyrk Apr 17, 2023
dc884eb
comments
deansheather Apr 17, 2023
dbbd2ba
comments
deansheather Apr 17, 2023
1322f99
Use correct MW
Emyrk Apr 17, 2023
784fb68
Make gen/fmt/lint
Emyrk Apr 17, 2023
b72ef2f
Group vs route to fix swagger
Emyrk Apr 17, 2023
a4f205e
comments
deansheather Apr 17, 2023
8508138
comments
deansheather Apr 17, 2023
cfe484c
comments
deansheather Apr 17, 2023
d4d9bf9
tests for RequireAPIKeyOrWorkspaceProxyAuth
deansheather Apr 17, 2023
fdbd31e
tests for ExtractWorkspaceProxy
deansheather Apr 17, 2023
efef018
Merge branch 'main' into dreamteam/external_proxy
deansheather Apr 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add proxy token provider
  • Loading branch information
Emyrk committed Apr 6, 2023
commit 020b4b5155958c0b8799c186e7bf00ed79dc5521
4 changes: 3 additions & 1 deletion coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -338,12 +338,14 @@ func New(options *Options) *API {
AccessURL: api.AccessURL,
Hostname: api.AppHostname,
HostnameRegex: api.AppHostnameRegex,
DeploymentValues: options.DeploymentValues,
RealIPConfig: options.RealIPConfig,

SignedTokenProvider: api.WorkspaceAppsProvider,
WorkspaceConnCache: api.workspaceAgentCache,
AppSecurityKey: options.AppSecurityKey,

DisablePathApps: options.DeploymentValues.DisablePathApps.Value(),
SecureAuthCookie: options.DeploymentValues.SecureAuthCookie.Value(),
}

apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{
Expand Down
6 changes: 3 additions & 3 deletions coderd/httpmw/apikey.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func ExtractAPIKey(rw http.ResponseWriter, r *http.Request, cfg ExtractAPIKeyCon
return nil, nil, false
}

token := apiTokenFromRequest(r)
token := ApiTokenFromRequest(r)
if token == "" {
return optionalWrite(http.StatusUnauthorized, codersdk.Response{
Message: SignedOutErrorMessage,
Expand Down Expand Up @@ -376,14 +376,14 @@ func ExtractAPIKey(rw http.ResponseWriter, r *http.Request, cfg ExtractAPIKeyCon
return &key, &authz, true
}

// apiTokenFromRequest returns the api token from the request.
// ApiTokenFromRequest returns the api token from the request.
// Find the session token from:
// 1: The cookie
// 1: The devurl cookie
// 3: The old cookie
// 4. The coder_session_token query parameter
// 5. The custom auth header
func apiTokenFromRequest(r *http.Request) string {
func ApiTokenFromRequest(r *http.Request) string {
cookie, err := r.Cookie(codersdk.SessionTokenCookie)
if err == nil && cookie.Value != "" {
return cookie.Value
Expand Down
2 changes: 1 addition & 1 deletion coderd/httpmw/workspaceagent.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func ExtractWorkspaceAgent(db database.Store) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
tokenValue := apiTokenFromRequest(r)
tokenValue := ApiTokenFromRequest(r)
if tokenValue == "" {
httpapi.Write(ctx, rw, http.StatusUnauthorized, codersdk.Response{
Message: fmt.Sprintf("Cookie %q must be provided.", codersdk.SessionTokenCookie),
Expand Down
18 changes: 1 addition & 17 deletions coderd/workspaceapps/db.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,23 +55,7 @@ func NewDBTokenProvider(log slog.Logger, accessURL *url.URL, authz rbac.Authoriz
}

func (p *DBTokenProvider) TokenFromRequest(r *http.Request) (*SignedToken, bool) {
// Get the existing token from the request.
tokenCookie, err := r.Cookie(codersdk.DevURLSignedAppTokenCookie)
if err == nil {
token, err := p.SigningKey.VerifySignedToken(tokenCookie.Value)
if err == nil {
req := token.Request.Normalize()
err := req.Validate()
if err == nil {
// The request has a valid signed app token, which is a valid
// token signed by us. The caller must check that it matches
// the request.
return &token, true
}
}
}

return nil, false
return TokenFromRequest(r, p.SigningKey)
}

// ResolveRequest takes an app request, checks if it's valid and authenticated,
Expand Down
12 changes: 7 additions & 5 deletions coderd/workspaceapps/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,16 @@ type Server struct {
Hostname string
// HostnameRegex contains the regex version of Hostname as generated by
// httpapi.CompileHostnamePattern(). It MUST be set if Hostname is set.
HostnameRegex *regexp.Regexp
DeploymentValues *codersdk.DeploymentValues
RealIPConfig *httpmw.RealIPConfig
HostnameRegex *regexp.Regexp
RealIPConfig *httpmw.RealIPConfig

SignedTokenProvider SignedTokenProvider
WorkspaceConnCache *wsconncache.Cache
AppSecurityKey SecurityKey

DisablePathApps bool
SecureAuthCookie bool

websocketWaitMutex sync.Mutex
websocketWaitGroup sync.WaitGroup
}
Expand Down Expand Up @@ -120,7 +122,7 @@ func (s *Server) Attach(r chi.Router) {
// workspaceAppsProxyPath proxies requests to a workspace application
// through a relative URL path.
func (s *Server) workspaceAppsProxyPath(rw http.ResponseWriter, r *http.Request) {
if s.DeploymentValues.DisablePathApps.Value() {
if s.DisablePathApps {
site.RenderStaticErrorPage(rw, r, site.ErrorPageData{
Status: http.StatusUnauthorized,
Title: "Unauthorized",
Expand Down Expand Up @@ -385,7 +387,7 @@ func (s *Server) setWorkspaceAppCookie(rw http.ResponseWriter, r *http.Request,
MaxAge: 0,
HttpOnly: true,
SameSite: http.SameSiteLaxMode,
Secure: s.DeploymentValues.SecureAuthCookie.Value(),
Secure: s.SecureAuthCookie,
})

return true
Expand Down
22 changes: 22 additions & 0 deletions coderd/workspaceapps/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"net/http"
"time"

"github.com/go-jose/go-jose/v3"
"github.com/google/uuid"
"golang.org/x/xerrors"

"github.com/coder/coder/coderd/database"
"github.com/coder/coder/codersdk"
)

const (
Expand Down Expand Up @@ -217,3 +219,23 @@ func (k SecurityKey) DecryptAPIKey(encryptedAPIKey string) (string, error) {

return payload.APIKey, nil
}

func TokenFromRequest(r *http.Request, key SecurityKey) (*SignedToken, bool) {
// Get the existing token from the request.
tokenCookie, err := r.Cookie(codersdk.DevURLSignedAppTokenCookie)
if err == nil {
token, err := key.VerifySignedToken(tokenCookie.Value)
if err == nil {
req := token.Request.Normalize()
err := req.Validate()
if err == nil {
// The request has a valid signed app token, which is a valid
// token signed by us. The caller must check that it matches
// the request.
return &token, true
}
}
}

return nil, false
}
46 changes: 36 additions & 10 deletions enterprise/coderd/workspaceproxy_test.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
package coderd_test

import (
"net/http/httptest"
"testing"

"github.com/google/uuid"
"github.com/moby/moby/pkg/namesgenerator"
"github.com/stretchr/testify/require"

"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
"github.com/coder/coder/agent"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/coderd/database/dbtestutil"
"github.com/coder/coder/coderd/workspaceapps"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/codersdk/agentsdk"
"github.com/coder/coder/enterprise/coderd/coderdenttest"
"github.com/coder/coder/enterprise/coderd/license"
"github.com/coder/coder/enterprise/wsproxy/wsproxysdk"
Expand Down Expand Up @@ -92,7 +97,21 @@ func TestIssueSignedAppToken(t *testing.T) {
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
workspace.LatestBuild = build

// Connect an agent to the workspace
agentClient := agentsdk.New(client.URL)
agentClient.SetSessionToken(authToken)
agentCloser := agent.New(agent.Options{
Client: agentClient,
Logger: slogtest.Make(t, nil).Named("agent").Leveled(slog.LevelDebug),
})
defer func() {
_ = agentCloser.Close()
}()

coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)

ctx := testutil.Context(t, testutil.WaitLong)
proxyRes, err := client.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
Expand All @@ -119,16 +138,23 @@ func TestIssueSignedAppToken(t *testing.T) {
require.Error(t, err)
})

goodRequest := wsproxysdk.IssueSignedAppTokenRequest{
AppRequest: workspaceapps.Request{
BasePath: "/app",
AccessMethod: workspaceapps.AccessMethodTerminal,
WorkspaceAndAgent: workspace.ID.String(),
AgentNameOrID: build.Resources[0].Agents[0].ID.String(),
},
SessionToken: client.SessionToken(),
}
t.Run("OK", func(t *testing.T) {
_, err = proxyClient.IssueSignedAppToken(ctx, wsproxysdk.IssueSignedAppTokenRequest{
AppRequest: workspaceapps.Request{
BasePath: "/app",
AccessMethod: workspaceapps.AccessMethodTerminal,
UsernameOrID: user.UserID.String(),
WorkspaceAndAgent: workspace.ID.String(),
},
SessionToken: client.SessionToken(),
})
_, err = proxyClient.IssueSignedAppToken(ctx, goodRequest)
require.NoError(t, err)
})

t.Run("OKHTML", func(t *testing.T) {
rw := httptest.NewRecorder()
_, ok := proxyClient.IssueSignedAppTokenHTML(ctx, rw, goodRequest)
require.True(t, ok, "expected true")
})
}
42 changes: 42 additions & 0 deletions enterprise/wsproxy/mw.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package wsproxy

import (
"context"
"fmt"
"net/http"

"github.com/coder/coder/coderd/httpapi"
"github.com/coder/coder/coderd/httpmw"
"github.com/coder/coder/codersdk"
)

type userTokenKey struct{}

// UserSessionToken returns session token from ExtractSessionTokenMW
func UserSessionToken(r *http.Request) string {
key, ok := r.Context().Value(userTokenKey{}).(string)
if !ok {
panic("developer error: ExtractSessionTokenMW middleware not provided")
}
return key
}

func ExtractSessionTokenMW() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
token := httpmw.ApiTokenFromRequest(r)
if token == "" {
// TODO: If this is empty, we should attempt to smuggle their
// token from the primary. If the user is not logged in there
// they should be redirected to a login page.
httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{
Message: httpmw.SignedOutErrorMessage,
Detail: fmt.Sprintf("Cookie %q or query parameter must be provided.", codersdk.SessionTokenCookie),
})
return
}
ctx := context.WithValue(r.Context(), userTokenKey{}, token)
next.ServeHTTP(rw, r.WithContext(ctx))
})
}
}
34 changes: 23 additions & 11 deletions enterprise/wsproxy/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/trace"
"golang.org/x/xerrors"

"cdr.dev/slog"
"github.com/coder/coder/buildinfo"
Expand All @@ -19,6 +20,7 @@ import (
"github.com/coder/coder/coderd/workspaceapps"
"github.com/coder/coder/coderd/wsconncache"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/enterprise/wsproxy/wsproxysdk"
)

type Options struct {
Expand Down Expand Up @@ -83,19 +85,22 @@ type Server struct {
cancel context.CancelFunc
}

func New(opts *Options) *Server {
func New(opts *Options) (*Server, error) {
if opts.PrometheusRegistry == nil {
opts.PrometheusRegistry = prometheus.NewRegistry()
}

client := codersdk.New(opts.PrimaryAccessURL)
client := wsproxysdk.New(opts.PrimaryAccessURL)
// TODO: @emyrk we need to implement some form of authentication for the
// external proxy to the the primary. This allows us to make workspace
// connections.
// Ideally we reuse the same client as the cli, but this can be changed.
// If the auth fails, we need some logic to retry and make sure this client
// is always authenticated and usable.
client.SetSessionToken("fake-token")
err := client.SetSessionToken("fake-token")
if err != nil {
return nil, xerrors.Errorf("set client token: %w", err)
}

r := chi.NewRouter()
ctx, cancel := context.WithCancel(context.Background())
Expand All @@ -116,13 +121,19 @@ func New(opts *Options) *Server {
AccessURL: opts.AccessURL,
Hostname: opts.AppHostname,
HostnameRegex: opts.AppHostnameRegex,
// TODO: @emyrk We should reduce the options passed in here.
DeploymentValues: nil,
RealIPConfig: opts.RealIPConfig,
// TODO: @emyrk we need to implement this for external token providers.
SignedTokenProvider: nil,
WorkspaceConnCache: wsconncache.New(s.DialWorkspaceAgent, 0),
AppSecurityKey: opts.AppSecurityKey,
RealIPConfig: opts.RealIPConfig,
SignedTokenProvider: &ProxyTokenProvider{
DashboardURL: opts.PrimaryAccessURL,
Client: client,
SecurityKey: s.Options.AppSecurityKey,
Logger: s.Logger.Named("proxy_token_provider"),
},
WorkspaceConnCache: wsconncache.New(s.DialWorkspaceAgent, 0),
AppSecurityKey: opts.AppSecurityKey,

// TODO: We need to pass some deployment values to here
DisablePathApps: false,
SecureAuthCookie: false,
}

// Routes
Expand All @@ -137,6 +148,7 @@ func New(opts *Options) *Server {
httpmw.ExtractRealIP(s.Options.RealIPConfig),
httpmw.Logger(s.Logger),
httpmw.Prometheus(s.PrometheusRegistry),
ExtractSessionTokenMW(),

// SubdomainAppMW is a middleware that handles all requests to the
// subdomain based workspace apps.
Expand Down Expand Up @@ -171,7 +183,7 @@ func New(opts *Options) *Server {

// TODO: @emyrk Buildinfo and healthz routes.

return s
return s, nil
}

func (s *Server) Close() error {
Expand Down
Loading