Add user local provisioner daemons

coder · kylecarbs · Nov 16, 2022 · Oct 25, 2022 · Nov 7, 2022 · Nov 7, 2022
commit e97287f8b33e6da5870ffd0cff6c0403bb8f3509
diff --git a/coderd/autobuild/executor/lifecycle_executor.go b/coderd/autobuild/executor/lifecycle_executor.go
@@ -277,6 +277,7 @@ func build(ctx context.Context, store database.Store, workspace database.Workspa
 		Type:           database.ProvisionerJobTypeWorkspaceBuild,
 		StorageMethod:  priorJob.StorageMethod,
 		FileID:         priorJob.FileID,
+		Tags:           priorJob.Tags,
 		Input:          input,
 	})
 	if err != nil {

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -36,6 +37,7 @@ import (
 	"github.com/coder/coder/coderd/audit"
 	"github.com/coder/coder/coderd/awsidentity"
 	"github.com/coder/coder/coderd/database"
+	"github.com/coder/coder/coderd/database/dbtype"
 	"github.com/coder/coder/coderd/gitauth"
 	"github.com/coder/coder/coderd/gitsshkey"
 	"github.com/coder/coder/coderd/httpapi"
@@ -659,11 +661,19 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client pro
 		CreatedAt:    database.Now(),
 		Name:         name,
 		Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform},
+		Tags: dbtype.Map{
+			provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+		},
 	})
 	if err != nil {
 		return nil, xerrors.Errorf("insert provisioner daemon %q: %w", name, err)
 	}
 
+	tags, err := json.Marshal(daemon.Tags)
+	if err != nil {
+		return nil, xerrors.Errorf("marshal tags: %w", err)
+	}
+
 	mux := drpcmux.New()
 	err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdserver.Server{
 		AccessURL:    api.AccessURL,
@@ -672,6 +682,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client pro
 		Pubsub:       api.Pubsub,
 		Provisioners: daemon.Provisioners,
 		Telemetry:    api.Telemetry,
+		Tags:         tags,
 		Logger:       api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)),
 	})
 	if err != nil {

diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go
@@ -3,7 +3,7 @@ package databasefake
 import (
 	"context"
 	"database/sql"
-	"reflect"
+	"encoding/json"
 	"sort"
 	"strings"
 	"sync"
@@ -147,7 +147,27 @@ func (q *fakeQuerier) AcquireProvisionerJob(_ context.Context, arg database.Acqu
 		if !found {
 			continue
 		}
-		if !reflect.DeepEqual(arg.Tags, provisionerJob.Tags) {
+		tags := map[string]string{}
+		if arg.Tags != nil {
+			err := json.Unmarshal(arg.Tags, &tags)
+			if err != nil {
+				return provisionerJob, xerrors.Errorf("unmarshal: %w", err)
+			}
+		}
+
+		missing := false
+		for key, value := range provisionerJob.Tags {
+			provided, found := tags[key]
+			if !found {
+				missing = true
+				break
+			}
+			if provided != value {
+				missing = true
+				break
+			}
+		}
+		if missing {
 			continue
 		}
 		provisionerJob.StartedAt = arg.StartedAt
@@ -2291,6 +2311,7 @@ func (q *fakeQuerier) InsertProvisionerJob(_ context.Context, arg database.Inser
 		FileID:         arg.FileID,
 		Type:           arg.Type,
 		Input:          arg.Input,
+		Tags:           arg.Tags,
 	}
 	q.provisionerJobs = append(q.provisionerJobs, job)
 	return job, nil

diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql
diff --git a/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql b/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql
@@ -1,2 +1,5 @@
 ALTER TABLE provisioner_daemons ADD COLUMN tags jsonb NOT NULL DEFAULT '{}';
-ALTER TABLE provisioner_jobs ADD COLUMN tags jsonb NOT NULL DEFAULT '{}';
+
+-- We must add the organization scope by default, otherwise pending jobs
+-- could be provisioned on new daemons that don't match the tags.
+ALTER TABLE provisioner_jobs ADD COLUMN tags jsonb NOT NULL DEFAULT '{"scope":"organization"}';
diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go
@@ -32,6 +32,7 @@ type Server struct {
 	ID           uuid.UUID
 	Logger       slog.Logger
 	Provisioners []database.ProvisionerType
+	Tags         json.RawMessage
 	Database     database.Store
 	Pubsub       database.Pubsub
 	Telemetry    telemetry.Reporter
@@ -50,6 +51,7 @@ func (server *Server) AcquireJob(ctx context.Context, _ *proto.Empty) (*proto.Ac
 			Valid: true,
 		},
 		Types: server.Provisioners,
+		Tags:  server.Tags,
 	})
 	if errors.Is(err, sql.ErrNoRows) {
 		// The provisioner daemon assumes no jobs are available if

diff --git a/coderd/provisionerdserver/provisionertags.go b/coderd/provisionerdserver/provisionertags.go
@@ -0,0 +1,33 @@
+package provisionerdserver
+
+import "github.com/google/uuid"
+
+const (
+	TagScope = "scope"
+	TagOwner = "owner"
+
+	ScopeUser         = "user"
+	ScopeOrganization = "organization"
+)
+
+// MutateTags adjusts the "owner" tag dependent on the "scope".
+// If the scope is "user", the "owner" is changed to the user ID.
+// This is for user-scoped provisioner daemons, where users should
+// own their own operations.
+func MutateTags(userID uuid.UUID, tags map[string]string) map[string]string {
+	if tags == nil {
+		tags = map[string]string{}
+	}
+	_, ok := tags[TagScope]
+	if !ok {
+		tags[TagScope] = ScopeOrganization
+	}
+	switch tags[TagScope] {
+	case ScopeUser:
+		tags[TagOwner] = userID.String()
+	case ScopeOrganization:
+	default:
+		tags[TagScope] = ScopeOrganization
+	}
+	return tags
+}
diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go
@@ -311,6 +311,7 @@ func convertProvisionerJob(provisionerJob database.ProvisionerJob) codersdk.Prov
 		CreatedAt: provisionerJob.CreatedAt,
 		Error:     provisionerJob.Error.String,
 		FileID:    provisionerJob.FileID,
+		Tags:      provisionerJob.Tags,
 	}
 	// Applying values optional to the struct.
 	if provisionerJob.StartedAt.Valid {

diff --git a/coderd/templateversions.go b/coderd/templateversions.go
@@ -288,6 +288,8 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques
 		FileID:         job.FileID,
 		Type:           database.ProvisionerJobTypeTemplateVersionDryRun,
 		Input:          input,
+		// Copy tags from the previous run.
+		Tags: job.Tags,
 	})
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
@@ -717,6 +719,9 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 		return
 	}
 
+	// Ensures the "owner" is properly applied.
+	tags := provisionerdserver.MutateTags(apiKey.UserID, req.ProvisionerTags)
+
 	file, err := api.Database.GetFileByID(ctx, req.FileID)
 	if errors.Is(err, sql.ErrNoRows) {
 		httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
@@ -815,6 +820,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 			FileID:         file.ID,
 			Type:           database.ProvisionerJobTypeTemplateVersionImport,
 			Input:          []byte{'{', '}'},
+			Tags:           tags,
 		})
 		if err != nil {
 			return xerrors.Errorf("insert provisioner job: %w", err)

diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go
@@ -13,6 +13,7 @@ import (
 	"github.com/coder/coder/coderd/audit"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/coderd/database"
+	"github.com/coder/coder/coderd/provisionerdserver"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/provisioner/echo"
 	"github.com/coder/coder/provisionersdk/proto"
@@ -122,6 +123,7 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) {
 		})
 		require.NoError(t, err)
 		require.Equal(t, "bananas", version.Name)
+		require.Equal(t, provisionerdserver.ScopeOrganization, version.Job.Tags[provisionerdserver.TagScope])
 
 		require.Len(t, auditor.AuditLogs, 1)
 		assert.Equal(t, database.AuditActionCreate, auditor.AuditLogs[0].Action)

diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go
@@ -428,6 +428,8 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	tags := provisionerdserver.MutateTags(workspace.OwnerID, templateVersionJob.Tags)
+
 	// Store prior build number to compute new build number
 	var priorBuildNum int32
 	priorHistory, err := api.Database.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID)
@@ -513,6 +515,7 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
 			StorageMethod:  templateVersionJob.StorageMethod,
 			FileID:         templateVersionJob.FileID,
 			Input:          input,
+			Tags:           tags,
 		})
 		if err != nil {
 			return xerrors.Errorf("insert provisioner job: %w", err)

diff --git a/coderd/workspaces.go b/coderd/workspaces.go
@@ -428,6 +428,8 @@ func (api *API) postWorkspacesByOrganization(rw http.ResponseWriter, r *http.Req
 		return
 	}
 
+	tags := provisionerdserver.MutateTags(user.ID, templateVersionJob.Tags)
+
 	var (
 		provisionerJob database.ProvisionerJob
 		workspaceBuild database.WorkspaceBuild
@@ -490,6 +492,7 @@ func (api *API) postWorkspacesByOrganization(rw http.ResponseWriter, r *http.Req
 			StorageMethod:  templateVersionJob.StorageMethod,
 			FileID:         templateVersionJob.FileID,
 			Input:          input,
+			Tags:           tags,
 		})
 		if err != nil {
 			return xerrors.Errorf("insert provisioner job: %w", err)

diff --git a/codersdk/organizations.go b/codersdk/organizations.go
@@ -36,11 +36,12 @@ type Organization struct {
 type CreateTemplateVersionRequest struct {
 	Name string `json:"name,omitempty" validate:"omitempty,template_name"`
 	// TemplateID optionally associates a version with a template.
-	TemplateID uuid.UUID `json:"template_id,omitempty"`
+	TemplateID      uuid.UUID                `json:"template_id,omitempty"`
+	StorageMethod   ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"`
+	FileID          uuid.UUID                `json:"file_id" validate:"required"`
+	Provisioner     ProvisionerType          `json:"provisioner" validate:"oneof=terraform echo,required"`
+	ProvisionerTags map[string]string        `json:"tags"`
 
-	StorageMethod ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"`
-	FileID        uuid.UUID                `json:"file_id" validate:"required"`
-	Provisioner   ProvisionerType          `json:"provisioner" validate:"oneof=terraform echo,required"`
 	// ParameterValues allows for additional parameters to be provided
 	// during the dry-run provision stage.
 	ParameterValues []CreateParameterRequest `json:"parameter_values,omitempty"`

diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go
@@ -76,6 +76,7 @@ type ProvisionerJob struct {
 	Status      ProvisionerJobStatus `json:"status"`
 	WorkerID    *uuid.UUID           `json:"worker_id,omitempty"`
 	FileID      uuid.UUID            `json:"file_id"`
+	Tags        map[string]string    `json:"tags"`
 }
 
 type ProvisionerJobLog struct {
@@ -166,10 +167,6 @@ func (c *Client) provisionerJobLogsAfter(ctx context.Context, path string, after
 	}), nil
 }
 
-type CreateProvisionerDaemonRequest struct {
-	Name string `json:"name" validate:"required"`
-}
-
 // ListenProvisionerDaemon returns the gRPC service for a provisioner daemon implementation.
 func (c *Client) ServeProvisionerDaemon(ctx context.Context, organization uuid.UUID, provisioners []ProvisionerType, tags map[string]string) (proto.DRPCProvisionerDaemonClient, error) {
 	serverURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/organizations/%s/provisionerdaemons/serve", organization))

diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go
@@ -91,7 +91,10 @@ func New(ctx context.Context, options *Options) (*API, error) {
 			})
 		})
 		r.Route("/organizations/{organization}/provisionerdaemons", func(r chi.Router) {
-			r.Use(apiKeyMiddleware)
+			r.Use(
+				apiKeyMiddleware,
+				httpmw.ExtractOrganizationParam(api.Database),
+			)
 			r.Get("/", api.provisionerDaemons)
 			r.Get("/serve", api.provisionerDaemonServe)
 		})

diff --git a/enterprise/coderd/coderdenttest/coderdenttest_test.go b/enterprise/coderd/coderdenttest/coderdenttest_test.go
@@ -47,6 +47,8 @@ func TestAuthorizeAllEndpoints(t *testing.T) {
 	a.URLParams["{groupName}"] = group.Name
 
 	skipRoutes, assertRoute := coderdtest.AGPLRoutes(a)
+	skipRoutes["GET:/api/v2/organizations/{organization}/provisionerdaemons/serve"] = "This route checks for RBAC dependent on input parameters!"
+
 	assertRoute["GET:/api/v2/entitlements"] = coderdtest.RouteCheck{
 		NoAuthorize: true,
 	}
@@ -84,6 +86,14 @@ func TestAuthorizeAllEndpoints(t *testing.T) {
 		AssertAction: rbac.ActionRead,
 		AssertObject: groupObj,
 	}
+	assertRoute["GET:/api/v2/organizations/{organization}/provisionerdaemons"] = coderdtest.RouteCheck{
+		AssertAction: rbac.ActionRead,
+		AssertObject: rbac.ResourceProvisionerDaemon,
+	}
+	assertRoute["GET:/api/v2/organizations/{organization}/provisionerdaemons"] = coderdtest.RouteCheck{
+		AssertAction: rbac.ActionRead,
+		AssertObject: rbac.ResourceProvisionerDaemon,
+	}
 	assertRoute["GET:/api/v2/groups/{group}"] = coderdtest.RouteCheck{
 		AssertAction: rbac.ActionRead,
 		AssertObject: groupObj,

diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go
@@ -2,6 +2,7 @@ package coderd
 
 import (
 	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -30,6 +31,11 @@ import (
 
 func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
+	org := httpmw.OrganizationParam(r)
+	if !api.Authorize(r, rbac.ActionRead, rbac.ResourceProvisionerDaemon.InOrg(org.ID)) {
+		httpapi.Forbidden(rw)
+		return
+	}
 	daemons, err := api.Database.GetProvisionerDaemons(ctx)
 	if errors.Is(err, sql.ErrNoRows) {
 		err = nil
@@ -97,8 +103,15 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
 	// for jobs that they own, but only authorized users can create
 	// globally scoped provisioners that attach to all jobs.
 	apiKey := httpmw.APIKey(r)
-	if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) {
-		tags["owner"] = apiKey.UserID.String()
+	tags = provisionerdserver.MutateTags(apiKey.UserID, tags)
+
+	if tags[provisionerdserver.TagScope] == provisionerdserver.ScopeOrganization {
+		if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) {
+			httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{
+				Message: "You aren't allowed to create provisioner daemons for the organization.",
+			})
+			return
+		}
 	}
 
 	name := namesgenerator.GetRandomName(1)
@@ -117,6 +130,15 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	rawTags, err := json.Marshal(daemon.Tags)
+	if err != nil {
+		httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Internal error marshaling daemon tags.",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
 	api.AGPL.WebsocketWaitMutex.Lock()
 	api.AGPL.WebsocketWaitGroup.Add(1)
 	api.AGPL.WebsocketWaitMutex.Unlock()
@@ -155,6 +177,7 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
 		Provisioners: daemon.Provisioners,
 		Telemetry:    api.Telemetry,
 		Logger:       api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)),
+		Tags:         rawTags,
 	})
 	if err != nil {
 		_ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("drpc register provisioner daemon: %s", err))