fixup! feat: add user maintenance schedule for max_ttl autostop

coder · deansheather · Jul 20, 2023 · Jun 20, 2023 · Jun 20, 2023 · Jun 21, 2023
commit 54d939adb2e3213a8da101342bd4bc2b31ee85a7
diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"reflect"
@@ -950,7 +951,36 @@ func (server *Server) CompleteJob(ctx context.Context, completed *proto.Complete
 				// Round the max deadline up to the nearest occurrence of the
 				// user's maintenance schedule. This ensures that workspaces
 				// can't be force-stopped due to max TTL during business hours.
-				maxDeadline = userMaintenanceSchedule.Schedule.Next(maxDeadline)
+
+				// Get the schedule occurrence that happens right before, during
+				// or after the max deadline.
+				// TODO: change to the maintenance window BEFORE max TTL
+				scheduleDur := userMaintenanceSchedule.Duration
+				if scheduleDur > 1*time.Hour {
+					// Allow a 15 minute buffer when possible so we're not too
+					// constrained with the autostop time.
+					scheduleDur -= 15 * time.Minute
+				}
+				windowStart := userMaintenanceSchedule.Schedule.Next(maxDeadline.Add(scheduleDur))
+
+				// Get the window of time that the workspace can be stopped in.
+				// This must be between windowStart and windowEnd, and also must
+				// be after the current max deadline.
+				minTime := maxDeadline
+				if windowStart.After(minTime) {
+					minTime = windowStart
+				}
+				maxTime := windowStart.Add(scheduleDur)
+				if minTime.After(maxTime) {
+					// TODO: remove this panic once we have good tests, and add
+					// a sensible fallback instead
+					panic("minTime is after maxTime")
+				}
+
+				// Pick a random time between minTime and maxTime.
+				actualDur := maxTime.Sub(minTime)
+				jitter := time.Duration(rand.Int63n(int64(actualDur)))
+				maxDeadline = minTime.Add(jitter)
 			}
 
 			err = db.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{

diff --git a/coderd/schedule/user.go b/coderd/schedule/user.go
@@ -53,6 +53,7 @@ func (*agplUserMaintenanceScheduleStore) GetUserMaintenanceScheduleOptions(_ con
 	return UserMaintenanceScheduleOptions{
 		Schedule: nil,
 		UserSet:  false,
+		Duration: 0,
 	}, nil
 }
 
@@ -61,5 +62,6 @@ func (*agplUserMaintenanceScheduleStore) SetUserMaintenanceScheduleOptions(_ con
 	return UserMaintenanceScheduleOptions{
 		Schedule: nil,
 		UserSet:  false,
+		Duration: 0,
 	}, nil
 }
diff --git a/codersdk/deployment.go b/codersdk/deployment.go
@@ -166,6 +166,7 @@ type DeploymentValues struct {
 	WgtunnelHost                    clibase.String                  `json:"wgtunnel_host,omitempty" typescript:",notnull"`
 	DisableOwnerWorkspaceExec       clibase.Bool                    `json:"disable_owner_workspace_exec,omitempty" typescript:",notnull"`
 	ProxyHealthStatusInterval       clibase.Duration                `json:"proxy_health_status_interval,omitempty" typescript:",notnull"`
+	UserMaintenanceSchedule         UserMaintenanceScheduleConfig   `json:"user_maintenance_schedule,omitempty" typescript:",notnull"`
 
 	Config      clibase.YAMLConfigPath `json:"config,omitempty" typescript:",notnull"`
 	WriteConfig clibase.Bool           `json:"write_config,omitempty" typescript:",notnull"`
@@ -337,6 +338,11 @@ type DangerousConfig struct {
 	AllowAllCors                clibase.Bool `json:"allow_all_cors" typescript:",notnull"`
 }
 
+type UserMaintenanceScheduleConfig struct {
+	DefaultSchedule clibase.String   `json:"default_schedule" typescript:",notnull"`
+	WindowDuration  clibase.Duration `json:"window_duration" typescript:",notnull"`
+}
+
 const (
 	annotationEnterpriseKey = "enterprise"
 	annotationSecretKey     = "secret"
@@ -460,6 +466,11 @@ when required by your organization's security policy.`,
 			Description: `Tune the behavior of the provisioner, which is responsible for creating, updating, and deleting workspace resources.`,
 			YAML:        "provisioning",
 		}
+		deploymentGroupUserMaintenanceSchedule = clibase.Group{
+			Name:        "User Maintenance Schedule",
+			Description: "Allow users to set maintenance schedules each day for workspaces to avoid workspaces stopping during the day due to template max TTL.",
+			YAML:        "user-maintenance-schedule",
+		}
 		deploymentGroupDangerous = clibase.Group{
 			Name: "⚠️ Dangerous",
 			YAML: "dangerous",
@@ -1521,6 +1532,26 @@ Write out the current server config as YAML to stdout.`,
 			Group:       &deploymentGroupNetworkingHTTP,
 			YAML:        "proxyHealthInterval",
 		},
+		{
+			Name:        "Default Maintenance Schedule",
+			Description: "The default daily cron schedule applied to users that haven't set a custom maintenance schedule themselves. The maintenance schedule determines when workspaces will be force stopped due to the template's max TTL, and will round the max TTL up to be within the user's maintenance window (or default). The format is the same as the standard cron format, but the day-of-month, month and day-of-week must be *. Only one hour and minute can be specified (ranges or comma separated values are not supported).",
+			Flag:        "default-maintenance-schedule",
+			Env:         "CODER_MAINTENANCE_DEFAULT_SCHEDULE",
+			Default:     "",
+			Value:       &c.UserMaintenanceSchedule.DefaultSchedule,
+			Group:       &deploymentGroupUserMaintenanceSchedule,
+			YAML:        "defaultMaintenanceSchedule",
+		},
+		{
+			Name:        "Maintenance Window Duration",
+			Description: "The duration of maintenance windows when triggered by cron. Workspaces can only be stopped due to max TTL during this window. Must be at least 1 hour.",
+			Flag:        "maintenance-window-duration",
+			Env:         "CODER_MAINTENANCE_WINDOW_DURATION",
+			Default:     (4 * time.Hour).String(),
+			Value:       &c.UserMaintenanceSchedule.DefaultSchedule,
+			Group:       &deploymentGroupUserMaintenanceSchedule,
+			YAML:        "maintenanceWindowDuration",
+		},
 	}
 	return opts
 }

diff --git a/enterprise/cli/server.go b/enterprise/cli/server.go
@@ -57,14 +57,16 @@ func (r *RootCmd) server() *clibase.Cmd {
 		options.TrialGenerator = trialer.New(options.Database, "https://v2-licensor.coder.com/trial", coderd.Keys)
 
 		o := &coderd.Options{
-			AuditLogging:           true,
-			BrowserOnly:            options.DeploymentValues.BrowserOnly.Value(),
-			SCIMAPIKey:             []byte(options.DeploymentValues.SCIMAPIKey.Value()),
-			RBAC:                   true,
-			DERPServerRelayAddress: options.DeploymentValues.DERP.Server.RelayURL.String(),
-			DERPServerRegionID:     int(options.DeploymentValues.DERP.Server.RegionID.Value()),
-			Options:                options,
-			ProxyHealthInterval:    options.DeploymentValues.ProxyHealthStatusInterval.Value(),
+			Options:                        options,
+			AuditLogging:                   true,
+			BrowserOnly:                    options.DeploymentValues.BrowserOnly.Value(),
+			SCIMAPIKey:                     []byte(options.DeploymentValues.SCIMAPIKey.Value()),
+			RBAC:                           true,
+			DERPServerRelayAddress:         options.DeploymentValues.DERP.Server.RelayURL.String(),
+			DERPServerRegionID:             int(options.DeploymentValues.DERP.Server.RegionID.Value()),
+			ProxyHealthInterval:            options.DeploymentValues.ProxyHealthStatusInterval.Value(),
+			DefaultUserMaintenanceSchedule: options.DeploymentValues.UserMaintenanceSchedule.DefaultSchedule.Value(),
+			UserMaintenanceWindowDuration:  options.DeploymentValues.UserMaintenanceSchedule.WindowDuration.Value(),
 		}
 
 		api, err := coderd.New(ctx, o)

diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go
@@ -52,6 +52,10 @@ func New(ctx context.Context, options *Options) (_ *API, err error) {
 	if options.Options.Authorizer == nil {
 		options.Options.Authorizer = rbac.NewCachingAuthorizer(options.PrometheusRegistry)
 	}
+	if options.UserMaintenanceWindowDuration < time.Hour {
+		return nil, xerrors.Errorf("user maintenance window duration must be at least 1 hour")
+	}
+
 	ctx, cancelFunc := context.WithCancel(ctx)
 	api := &API{
 		ctx:    ctx,
@@ -303,6 +307,10 @@ type Options struct {
 	DERPServerRelayAddress string
 	DERPServerRegionID     int
 
+	// Used for user maintenance schedules.
+	DefaultUserMaintenanceSchedule string        // cron schedule, if empty user maintenance schedules are disabled
+	UserMaintenanceWindowDuration  time.Duration // how long each window should last
+
 	EntitlementsUpdateInterval time.Duration
 	ProxyHealthInterval        time.Duration
 	Keys                       map[string]ed25519.PublicKey
@@ -354,7 +362,7 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 			codersdk.FeatureTemplateRBAC:               api.RBAC,
 			codersdk.FeatureExternalProvisionerDaemons: true,
 			codersdk.FeatureAdvancedTemplateScheduling: true,
-			codersdk.FeatureUserMaintenanceSchedule:    true,
+			codersdk.FeatureUserMaintenanceSchedule:    api.DefaultUserMaintenanceSchedule != "",
 			codersdk.FeatureWorkspaceProxy:             true,
 		})
 	if err != nil {
@@ -423,10 +431,13 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 	}
 
 	if changed, enabled := featureChanged(codersdk.FeatureUserMaintenanceSchedule); changed {
-		if enabled {
-			// TODO: configurable default schedule
-			store := schedule.NewEnterpriseUserMaintenanceScheduleStore("CRON_TZ=UTC 0 0 * * *")
-			api.AGPL.UserMaintenanceScheduleStore.Store(&store)
+		if enabled && api.DefaultUserMaintenanceSchedule != "" {
+			store, err := schedule.NewEnterpriseUserMaintenanceScheduleStore(api.DefaultUserMaintenanceSchedule, api.UserMaintenanceWindowDuration)
+			if err != nil {
+				api.Logger.Error(ctx, "unable to set up enterprise user maintenance schedule store, maintenance schedules will not be applied", slog.Error(err))
+			} else {
+				api.AGPL.UserMaintenanceScheduleStore.Store(&store)
+			}
 		} else {
 			store := agplschedule.NewAGPLUserMaintenanceScheduleStore()
 			api.AGPL.UserMaintenanceScheduleStore.Store(&store)

diff --git a/enterprise/coderd/schedule/user.go b/enterprise/coderd/schedule/user.go
@@ -12,21 +12,35 @@ import (
 	agpl "github.com/coder/coder/coderd/schedule"
 )
 
-const userMaintenanceWindowDuration = 4 * time.Hour
-
 // enterpriseUserMaintenanceScheduleStore provides an
 // agpl.UserMaintenanceScheduleStore that has all fields implemented for
 // enterprise customers.
 type enterpriseUserMaintenanceScheduleStore struct {
 	defaultSchedule string
+	windowDuration  time.Duration
 }
 
 var _ agpl.UserMaintenanceScheduleStore = &enterpriseUserMaintenanceScheduleStore{}
 
-func NewEnterpriseUserMaintenanceScheduleStore(defaultSchedule string) agpl.UserMaintenanceScheduleStore {
-	return &enterpriseUserMaintenanceScheduleStore{
+func NewEnterpriseUserMaintenanceScheduleStore(defaultSchedule string, windowDuration time.Duration) (agpl.UserMaintenanceScheduleStore, error) {
+	if defaultSchedule == "" {
+		return nil, xerrors.Errorf("default schedule must be set")
+	}
+	if windowDuration < 1*time.Hour {
+		return nil, xerrors.Errorf("window duration must be greater than 1 hour")
+	}
+
+	s := &enterpriseUserMaintenanceScheduleStore{
 		defaultSchedule: defaultSchedule,
+		windowDuration:  windowDuration,
 	}
+
+	_, err := s.parseSchedule(defaultSchedule)
+	if err != nil {
+		return nil, xerrors.Errorf("parse default schedule: %w", err)
+	}
+
+	return s, nil
 }
 
 func (s *enterpriseUserMaintenanceScheduleStore) parseSchedule(rawSchedule string) (agpl.UserMaintenanceScheduleOptions, error) {
@@ -49,7 +63,7 @@ func (s *enterpriseUserMaintenanceScheduleStore) parseSchedule(rawSchedule strin
 	return agpl.UserMaintenanceScheduleOptions{
 		Schedule: sched,
 		UserSet:  userSet,
-		Duration: userMaintenanceWindowDuration,
+		Duration: s.windowDuration,
 	}, nil
 }
 

diff --git a/enterprise/coderd/users.go b/enterprise/coderd/users.go
@@ -25,7 +25,7 @@ func (api *API) userMaintenanceSchedule(rw http.ResponseWriter, r *http.Request)
 		user = httpmw.UserParam(r)
 	)
 
-	// Double query here cuz of the user param
+	// TODO: Double query here cuz of the user param
 	opts, err := (*api.UserMaintenanceScheduleStore.Load()).GetUserMaintenanceScheduleOptions(ctx, api.Database, user.ID)
 	if err != nil {
 		// TODO: some of these errors are related to bad syntax, would be nice