chore: Implement workspace proxy going away

When a workspace proxy shuts down, the health status of that proxy should immediately be updated. This is purely a courtesy and technically not required
coder · Emyrk · May 11, 2023 · May 8, 2023 · May 8, 2023 · May 8, 2023
commit 9c4de6ec30029a0b113a4b44b3b46cf8bed2238c
diff --git a/codersdk/workspaceproxy.go b/codersdk/workspaceproxy.go
@@ -15,9 +15,9 @@ import (
 type ProxyHealthStatus string
 
 const (
-	// ProxyReachable means the proxy access url is reachable and returns a healthy
+	// ProxyHealthy means the proxy access url is reachable and returns a healthy
 	// status code.
-	ProxyReachable ProxyHealthStatus = "reachable"
+	ProxyHealthy ProxyHealthStatus = "ok"
 	// ProxyUnreachable means the proxy access url is not responding.
 	ProxyUnreachable ProxyHealthStatus = "unreachable"
 	// ProxyUnhealthy means the proxy access url is responding, but there is some
@@ -110,6 +110,24 @@ func (c *Client) WorkspaceProxies(ctx context.Context) ([]WorkspaceProxy, error)
 	return proxies, json.NewDecoder(res.Body).Decode(&proxies)
 }
 
+func (c *Client) WorkspaceProxyByName(ctx context.Context, name string) (WorkspaceProxy, error) {
+	res, err := c.Request(ctx, http.MethodGet,
+		fmt.Sprintf("/api/v2/workspaceproxies/%s", name),
+		nil,
+	)
+	if err != nil {
+		return WorkspaceProxy{}, xerrors.Errorf("make request: %w", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return WorkspaceProxy{}, ReadBodyAsError(res)
+	}
+
+	var proxy WorkspaceProxy
+	return proxy, json.NewDecoder(res.Body).Decode(&proxy)
+}
+
 func (c *Client) DeleteWorkspaceProxyByName(ctx context.Context, name string) error {
 	res, err := c.Request(ctx, http.MethodDelete,
 		fmt.Sprintf("/api/v2/workspaceproxies/%s", name),

diff --git a/enterprise/cli/proxyserver.go b/enterprise/cli/proxyserver.go
@@ -249,6 +249,7 @@ func (*RootCmd) proxyServer() *clibase.Cmd {
 			if err != nil {
 				return xerrors.Errorf("create workspace proxy: %w", err)
 			}
+			closers.Add(func() { _ = proxy.Close() })
 
 			shutdownConnsCtx, shutdownConns := context.WithCancel(ctx)
 			defer shutdownConns()

diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go
@@ -113,13 +113,15 @@ func New(ctx context.Context, options *Options) (*API, error) {
 				)
 				r.Post("/issue-signed-app-token", api.workspaceProxyIssueSignedAppToken)
 				r.Post("/register", api.workspaceProxyRegister)
+				r.Post("/goingaway", api.workspaceProxyGoingAway)
 			})
 			r.Route("/{workspaceproxy}", func(r chi.Router) {
 				r.Use(
 					apiKeyMiddleware,
 					httpmw.ExtractWorkspaceProxyParam(api.Database),
 				)
 
+				r.Get("/", api.getWorkspaceProxy)
 				r.Delete("/", api.deleteWorkspaceProxy)
 			})
 		})
@@ -237,7 +239,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
 	if api.AGPL.Experiments.Enabled(codersdk.ExperimentMoons) {
 		// Proxy health is a moon feature.
 		api.ProxyHealth, err = proxyhealth.New(&proxyhealth.Options{
-			Interval:   time.Minute * 1,
+			Interval:   options.ProxyHealthInterval,
 			DB:         api.Database,
 			Logger:     options.Logger.Named("proxyhealth"),
 			Client:     api.HTTPClient,

diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go
@@ -48,6 +48,7 @@ type Options struct {
 	EntitlementsUpdateInterval time.Duration
 	SCIMAPIKey                 []byte
 	UserWorkspaceQuota         int
+	ProxyHealthInterval        time.Duration
 }
 
 // New constructs a codersdk client connected to an in-memory Enterprise API instance.
@@ -74,6 +75,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
 		Options:                    oop,
 		EntitlementsUpdateInterval: options.EntitlementsUpdateInterval,
 		Keys:                       Keys,
+		ProxyHealthInterval:        options.ProxyHealthInterval,
 	})
 	assert.NoError(t, err)
 	setHandler(coderAPI.AGPL.RootHandler)

diff --git a/enterprise/coderd/workspaceproxy.go b/enterprise/coderd/workspaceproxy.go
@@ -68,11 +68,7 @@ func (api *API) regions(rw http.ResponseWriter, r *http.Request) {
 				continue
 			}
 
-			health, ok := proxyHealth[proxy.ID]
-			if !ok {
-				health.Status = proxyhealth.Unknown
-			}
-
+			health := proxyHealth[proxy.ID]
 			regions = append(regions, codersdk.Region{
 				ID:               proxy.ID,
 				Name:             proxy.Name,
@@ -254,6 +250,24 @@ func (api *API) workspaceProxies(rw http.ResponseWriter, r *http.Request) {
 	httpapi.Write(ctx, rw, http.StatusOK, convertProxies(proxies, statues))
 }
 
+// @Summary Get workspace proxy
+// @ID get-workspace-proxy
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Enterprise
+// @Param workspaceproxy path string true "Proxy ID or name" format(uuid)
+// @Success 200 {object} codersdk.WorkspaceProxy
+// @Router /workspaceproxies/{workspaceproxy} [get]
+func (api *API) getWorkspaceProxy(rw http.ResponseWriter, r *http.Request) {
+	var (
+		ctx   = r.Context()
+		proxy = httpmw.WorkspaceProxyParam(r)
+	)
+
+	status := api.ProxyHealth.HealthStatus()[proxy.ID]
+	httpapi.Write(ctx, rw, http.StatusOK, convertProxy(proxy, status))
+}
+
 // @Summary Issue signed workspace app token
 // @ID issue-signed-workspace-app-token
 // @Security CoderSessionToken
@@ -371,6 +385,35 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
 	go api.forceWorkspaceProxyHealthUpdate(api.ctx)
 }
 
+// workspaceProxyGoingAway is used to tell coderd that the workspace proxy is
+// shutting down and going away. The main purpose of this function is for the
+// health status of the workspace proxy to be more quickly updated when we know
+// that the proxy is going to be unhealthy. This does not delete the workspace
+// or cause any other side effects.
+// If the workspace proxy comes back online, even without a register, it will
+// be found healthy again by the normal checks.
+// @Summary Workspace proxy going away
+// @ID workspace-proxy-going-away
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Enterprise
+// @Success 201 {object} codersdk.Response
+// @Router /workspaceproxies/me/goingaway [post]
+// @x-apidocgen {"skip": true}
+func (api *API) workspaceProxyGoingAway(rw http.ResponseWriter, r *http.Request) {
+	var (
+		ctx = r.Context()
+	)
+
+	// Force a health update to happen immediately. The proxy should
+	// not return a successful response if it is going away.
+	go api.forceWorkspaceProxyHealthUpdate(api.ctx)
+
+	httpapi.Write(ctx, rw, http.StatusOK, codersdk.Response{
+		Message: "OK",
+	})
+}
+
 // reconnectingPTYSignedToken issues a signed app token for use when connecting
 // to the reconnecting PTY websocket on an external workspace proxy. This is set
 // by the client as a query parameter when connecting.
@@ -476,6 +519,9 @@ func convertProxies(p []database.WorkspaceProxy, statuses map[uuid.UUID]proxyhea
 }
 
 func convertProxy(p database.WorkspaceProxy, status proxyhealth.ProxyStatus) codersdk.WorkspaceProxy {
+	if status.Status == "" {
+		status.Status = proxyhealth.Unknown
+	}
 	return codersdk.WorkspaceProxy{
 		ID:               p.ID,
 		Name:             p.Name,

diff --git a/enterprise/coderd/workspaceproxy_test.go b/enterprise/coderd/workspaceproxy_test.go
@@ -7,6 +7,7 @@ import (
 	"net/http/httputil"
 	"net/url"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/moby/moby/pkg/namesgenerator"
@@ -172,6 +173,66 @@ func TestRegions(t *testing.T) {
 		require.Error(t, err)
 		require.Empty(t, regions)
 	})
+
+	t.Run("GoingAway", func(t *testing.T) {
+		t.Parallel()
+
+		dv := coderdtest.DeploymentValues(t)
+		dv.Experiments = []string{
+			string(codersdk.ExperimentMoons),
+			"*",
+		}
+
+		db, pubsub := dbtestutil.NewDB(t)
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+
+		client, closer, api := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				AppHostname:      appHostname,
+				Database:         db,
+				Pubsub:           pubsub,
+				DeploymentValues: dv,
+			},
+			// The interval is set to 1 hour so the proxy health
+			// check will never happen manually. All checks will be
+			// forced updates.
+			ProxyHealthInterval: time.Hour,
+		})
+		t.Cleanup(func() {
+			_ = closer.Close()
+		})
+		_ = coderdtest.CreateFirstUser(t, client)
+		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspaceProxy: 1,
+			},
+		})
+
+		const proxyName = "testproxy"
+		proxy := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
+			Name: proxyName,
+		})
+		var _ = proxy
+
+		require.Eventuallyf(t, func() bool {
+			proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
+			if err != nil {
+				return false
+			}
+			return proxy.Status.Status == codersdk.ProxyHealthy
+		}, time.Second*10, time.Millisecond*100, "proxy never became healthy")
+
+		_ = proxy.Close()
+		// The proxy should tell the primary on close that is is no longer healthy.
+		require.Eventuallyf(t, func() bool {
+			proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
+			if err != nil {
+				return false
+			}
+			return proxy.Status.Status == codersdk.ProxyUnhealthy
+		}, time.Second*10, time.Millisecond*100, "proxy never became unhealthy after close")
+	})
 }
 
 func TestWorkspaceProxyCRUD(t *testing.T) {

diff --git a/enterprise/wsproxy/wsproxy.go b/enterprise/wsproxy/wsproxy.go
@@ -250,6 +250,11 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
 
 func (s *Server) Close() error {
 	s.cancel()
+	go func() {
+		// Do this in a go routine to not block the close. This is allowed
+		// to fail, it is just a courtesy to the dashboard.
+		_ = s.SDKClient.WorkspaceProxyGoingAway(context.Background())
+	}()
 	return s.AppServer.Close()
 }
 
@@ -279,6 +284,15 @@ func (s *Server) healthReport(rw http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 	var report codersdk.ProxyHealthReport
 
+	// This is to catch edge cases where the server is shutting down, but might
+	// still serve a web request that returns "healthy". This is mainly just for
+	// unit tests, as shutting down the test webserver is tied to the lifecycle
+	// of the test. In practice, the webserver is tied to the lifecycle of the
+	// app, so the webserver AND the proxy will be shut down at the same time.
+	if s.ctx.Err() != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, "workspace proxy in middle of shutting down")
+	}
+
 	// Hit the build info to do basic version checking.
 	primaryBuild, err := s.SDKClient.SDKClient.BuildInfo(ctx)
 	if err != nil {

diff --git a/enterprise/wsproxy/wsproxysdk/wsproxysdk.go b/enterprise/wsproxy/wsproxysdk/wsproxysdk.go
@@ -170,3 +170,19 @@ func (c *Client) RegisterWorkspaceProxy(ctx context.Context, req RegisterWorkspa
 	var resp RegisterWorkspaceProxyResponse
 	return resp, json.NewDecoder(res.Body).Decode(&resp)
 }
+
+func (c *Client) WorkspaceProxyGoingAway(ctx context.Context) error {
+	res, err := c.Request(ctx, http.MethodPost,
+		"/api/v2/workspaceproxies/me/goingaway",
+		nil,
+	)
+	if err != nil {
+		return xerrors.Errorf("make request: %w", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return codersdk.ReadBodyAsError(res)
+	}
+	return nil
+}