From 48fcf309cf1767b88bf87245884b42e85d35f516 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 25 Oct 2022 14:46:32 +0000 Subject: [PATCH 01/18] Start to port over provisioner daemons PR --- cli/server.go | 2 +- coderd/coderdtest/coderdtest.go | 2 +- coderd/database/databasefake/databasefake.go | 21 ++++ coderd/database/dump.sql | 3 +- .../000065_provisioner_daemon_auth.down.sql | 1 + .../000065_provisioner_daemon_auth.up.sql | 1 + coderd/database/models.go | 1 + coderd/database/querier.go | 1 + coderd/database/queries.sql.go | 38 +++++- .../database/queries/provisionerdaemons.sql | 13 +- coderd/httpmw/provisionerdaemon.go | 69 +++++++++++ coderd/provisionerdaemons.go | 117 +++++++++++++++++- codersdk/provisionerdaemons.go | 65 ++++++++++ 13 files changed, 323 insertions(+), 11 deletions(-) create mode 100644 coderd/database/migrations/000065_provisioner_daemon_auth.down.sql create mode 100644 coderd/database/migrations/000065_provisioner_daemon_auth.up.sql create mode 100644 coderd/httpmw/provisionerdaemon.go diff --git a/cli/server.go b/cli/server.go index 93de4e5e34960..6d476b247e2c6 100644 --- a/cli/server.go +++ b/cli/server.go @@ -907,7 +907,7 @@ func newProvisionerDaemon( }() provisioners[string(database.ProvisionerTypeEcho)] = proto.NewDRPCProvisionerClient(provisionersdk.Conn(echoClient)) } - return provisionerd.New(coderAPI.ListenProvisionerDaemon, &provisionerd.Options{ + return provisionerd.New(coderAPI.CreateInMemoryProvisionerDaemon, &provisionerd.Options{ Logger: logger, PollInterval: 500 * time.Millisecond, UpdateInterval: 500 * time.Millisecond, diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index 6758509635ffe..a27ad9b95cf43 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -320,7 +320,7 @@ func NewProvisionerDaemon(t *testing.T, coderAPI *coderd.API) io.Closer { assert.NoError(t, err) }() - closer := provisionerd.New(coderAPI.ListenProvisionerDaemon, &provisionerd.Options{ + closer := provisionerd.New(coderAPI.CreateInMemoryProvisionerDaemon, &provisionerd.Options{ Filesystem: fs, Logger: slogtest.Make(t, nil).Named("provisionerd").Leveled(slog.LevelDebug), PollInterval: 50 * time.Millisecond, diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index d772e6d81a85e..ea09e1051254f 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -1890,6 +1890,26 @@ func (q *fakeQuerier) GetProvisionerDaemonByID(_ context.Context, id uuid.UUID) return database.ProvisionerDaemon{}, sql.ErrNoRows } +func (q *fakeQuerier) GetProvisionerDaemonByAuthToken(_ context.Context, token uuid.NullUUID) (database.ProvisionerDaemon, error) { + if !token.Valid { + return database.ProvisionerDaemon{}, sql.ErrNoRows + } + + q.mutex.RLock() + defer q.mutex.RUnlock() + + for _, provisionerDaemon := range q.provisionerDaemons { + if !provisionerDaemon.AuthToken.Valid { + continue + } + if provisionerDaemon.AuthToken.UUID.String() != token.UUID.String() { + continue + } + return provisionerDaemon, nil + } + return database.ProvisionerDaemon{}, sql.ErrNoRows +} + func (q *fakeQuerier) GetProvisionerJobByID(_ context.Context, id uuid.UUID) (database.ProvisionerJob, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -2264,6 +2284,7 @@ func (q *fakeQuerier) InsertProvisionerDaemon(_ context.Context, arg database.In CreatedAt: arg.CreatedAt, Name: arg.Name, Provisioners: arg.Provisioners, + AuthToken: arg.AuthToken, } q.provisionerDaemons = append(q.provisionerDaemons, daemon) return daemon, nil diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index d601ac60cdd16..4864b34c74d01 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -265,7 +265,8 @@ CREATE TABLE provisioner_daemons ( updated_at timestamp with time zone, name character varying(64) NOT NULL, provisioners provisioner_type[] NOT NULL, - replica_id uuid + replica_id uuid, + auth_token uuid ); CREATE TABLE provisioner_job_logs ( diff --git a/coderd/database/migrations/000065_provisioner_daemon_auth.down.sql b/coderd/database/migrations/000065_provisioner_daemon_auth.down.sql new file mode 100644 index 0000000000000..fd5c4ad9ea6ee --- /dev/null +++ b/coderd/database/migrations/000065_provisioner_daemon_auth.down.sql @@ -0,0 +1 @@ +ALTER TABLE provisioner_daemons DROP COLUMN IF EXISTS auth_token; diff --git a/coderd/database/migrations/000065_provisioner_daemon_auth.up.sql b/coderd/database/migrations/000065_provisioner_daemon_auth.up.sql new file mode 100644 index 0000000000000..69c00e5375bfe --- /dev/null +++ b/coderd/database/migrations/000065_provisioner_daemon_auth.up.sql @@ -0,0 +1 @@ +ALTER TABLE provisioner_daemons ADD COLUMN IF NOT EXISTS auth_token uuid; diff --git a/coderd/database/models.go b/coderd/database/models.go index 83e4610c40455..0503ecdcf9261 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -521,6 +521,7 @@ type ProvisionerDaemon struct { Name string `db:"name" json:"name"` Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` + AuthToken uuid.NullUUID `db:"auth_token" json:"auth_token"` } type ProvisionerJob struct { diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 69eb998f59d57..d437bc2945def 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -65,6 +65,7 @@ type sqlcQuerier interface { GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUID) ([]ParameterSchema, error) GetParameterSchemasCreatedAfter(ctx context.Context, createdAt time.Time) ([]ParameterSchema, error) GetParameterValueByScopeAndName(ctx context.Context, arg GetParameterValueByScopeAndNameParams) (ParameterValue, error) + GetProvisionerDaemonByAuthToken(ctx context.Context, authToken uuid.NullUUID) (ProvisionerDaemon, error) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) (ProvisionerDaemon, error) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDaemon, error) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (ProvisionerJob, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index b99d651016fb3..b05a7b9b03ff5 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2192,9 +2192,33 @@ func (q *sqlQuerier) ParameterValues(ctx context.Context, arg ParameterValuesPar return items, nil } +const getProvisionerDaemonByAuthToken = `-- name: GetProvisionerDaemonByAuthToken :one +SELECT + id, created_at, updated_at, name, provisioners, replica_id, auth_token +FROM + provisioner_daemons +WHERE + auth_token = $1 +` + +func (q *sqlQuerier) GetProvisionerDaemonByAuthToken(ctx context.Context, authToken uuid.NullUUID) (ProvisionerDaemon, error) { + row := q.db.QueryRowContext(ctx, getProvisionerDaemonByAuthToken, authToken) + var i ProvisionerDaemon + err := row.Scan( + &i.ID, + &i.CreatedAt, + &i.UpdatedAt, + &i.Name, + pq.Array(&i.Provisioners), + &i.ReplicaID, + &i.AuthToken, + ) + return i, err +} + const getProvisionerDaemonByID = `-- name: GetProvisionerDaemonByID :one SELECT - id, created_at, updated_at, name, provisioners, replica_id + id, created_at, updated_at, name, provisioners, replica_id, auth_token FROM provisioner_daemons WHERE @@ -2211,13 +2235,14 @@ func (q *sqlQuerier) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) &i.Name, pq.Array(&i.Provisioners), &i.ReplicaID, + &i.AuthToken, ) return i, err } const getProvisionerDaemons = `-- name: GetProvisionerDaemons :many SELECT - id, created_at, updated_at, name, provisioners, replica_id + id, created_at, updated_at, name, provisioners, replica_id, auth_token FROM provisioner_daemons ` @@ -2238,6 +2263,7 @@ func (q *sqlQuerier) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDa &i.Name, pq.Array(&i.Provisioners), &i.ReplicaID, + &i.AuthToken, ); err != nil { return nil, err } @@ -2258,10 +2284,11 @@ INSERT INTO id, created_at, "name", - provisioners + provisioners, + auth_token ) VALUES - ($1, $2, $3, $4) RETURNING id, created_at, updated_at, name, provisioners, replica_id + ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at, name, provisioners, replica_id, auth_token ` type InsertProvisionerDaemonParams struct { @@ -2269,6 +2296,7 @@ type InsertProvisionerDaemonParams struct { CreatedAt time.Time `db:"created_at" json:"created_at"` Name string `db:"name" json:"name"` Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` + AuthToken uuid.NullUUID `db:"auth_token" json:"auth_token"` } func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProvisionerDaemonParams) (ProvisionerDaemon, error) { @@ -2277,6 +2305,7 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv arg.CreatedAt, arg.Name, pq.Array(arg.Provisioners), + arg.AuthToken, ) var i ProvisionerDaemon err := row.Scan( @@ -2286,6 +2315,7 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv &i.Name, pq.Array(&i.Provisioners), &i.ReplicaID, + &i.AuthToken, ) return i, err } diff --git a/coderd/database/queries/provisionerdaemons.sql b/coderd/database/queries/provisionerdaemons.sql index 30ff6d9d43eda..e5f2e1b173a86 100644 --- a/coderd/database/queries/provisionerdaemons.sql +++ b/coderd/database/queries/provisionerdaemons.sql @@ -18,10 +18,11 @@ INSERT INTO id, created_at, "name", - provisioners + provisioners, + auth_token ) VALUES - ($1, $2, $3, $4) RETURNING *; + ($1, $2, $3, $4, $5) RETURNING *; -- name: UpdateProvisionerDaemonByID :exec UPDATE @@ -31,3 +32,11 @@ SET provisioners = $3 WHERE id = $1; + +-- name: GetProvisionerDaemonByAuthToken :one +SELECT + * +FROM + provisioner_daemons +WHERE + auth_token = $1; diff --git a/coderd/httpmw/provisionerdaemon.go b/coderd/httpmw/provisionerdaemon.go new file mode 100644 index 0000000000000..0d7f9c6f9d544 --- /dev/null +++ b/coderd/httpmw/provisionerdaemon.go @@ -0,0 +1,69 @@ +package httpmw + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + + "github.com/google/uuid" + + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/httpapi" + "github.com/coder/coder/codersdk" +) + +type provisionerDaemonContextKey struct{} + +// ProvisionerDaemon returns the daemon from the ExtractProvisionerDaemon handler. +func ProvisionerDaemon(r *http.Request) database.ProvisionerDaemon { + user, ok := r.Context().Value(provisionerDaemonContextKey{}).(database.ProvisionerDaemon) + if !ok { + panic("developer error: provisioner daemon middleware not provided") + } + return user +} + +// ExtractProvisionerDaemon requires authentication using a valid provisioner token. +func ExtractProvisionerDaemon(db database.Store) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + cookie, err := r.Cookie(codersdk.SessionTokenKey) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ + Message: fmt.Sprintf("Cookie %q must be provided.", codersdk.SessionTokenKey), + }) + return + } + token, err := uuid.Parse(cookie.Value) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ + Message: "Provisioner token is invalid.", + }) + return + } + provisioner, err := db.GetProvisionerDaemonByAuthToken(r.Context(), uuid.NullUUID{ + Valid: true, + UUID: token, + }) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ + Message: "Provisioner token is invalid.", + }) + return + } + + httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error fetching provisioner daemon.", + Detail: err.Error(), + }) + return + } + + ctx := context.WithValue(r.Context(), provisionerDaemonContextKey{}, provisioner) + next.ServeHTTP(rw, r.WithContext(ctx)) + }) + } +} diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index adb5cb2edfc35..56469026c16b2 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -13,10 +13,12 @@ import ( "time" "github.com/google/uuid" + "github.com/hashicorp/yamux" "github.com/moby/moby/pkg/namesgenerator" "github.com/tabbed/pqtype" "golang.org/x/xerrors" protobuf "google.golang.org/protobuf/proto" + "nhooyr.io/websocket" "storj.io/drpc/drpcmux" "storj.io/drpc/drpcserver" @@ -24,6 +26,7 @@ import ( "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/httpapi" + "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/parameter" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/coderd/telemetry" @@ -61,9 +64,103 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, daemons) } -// ListenProvisionerDaemon is an in-memory connection to a provisionerd. Useful when starting coderd and provisionerd +// Serves the provisioner daemon protobuf API over a WebSocket. +func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request) { + daemon := httpmw.ProvisionerDaemon(r) + api.Logger.Warn(r.Context(), "daemon connected", slog.F("daemon", daemon.Name)) + + api.websocketWaitMutex.Lock() + api.websocketWaitGroup.Add(1) + api.websocketWaitMutex.Unlock() + defer api.websocketWaitGroup.Done() + + conn, err := websocket.Accept(rw, r, &websocket.AcceptOptions{ + // Need to disable compression to avoid a data-race. + CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ + Message: "Internal error accepting websocket connection.", + Detail: err.Error(), + }) + return + } + // Align with the frame size of yamux. + conn.SetReadLimit(256 * 1024) + + // Multiplexes the incoming connection using yamux. + // This allows multiple function calls to occur over + // the same connection. + config := yamux.DefaultConfig() + config.LogOutput = io.Discard + session, err := yamux.Server(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), config) + if err != nil { + _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("multiplex server: %s", err)) + return + } + mux := drpcmux.New() + err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdServer{ + AccessURL: api.AccessURL, + ID: daemon.ID, + Database: api.Database, + Pubsub: api.Pubsub, + Provisioners: daemon.Provisioners, + Telemetry: api.Telemetry, + Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), + }) + if err != nil { + _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("drpc register provisioner daemon: %s", err)) + return + } + server := drpcserver.NewWithOptions(mux, drpcserver.Options{ + Log: func(err error) { + if xerrors.Is(err, io.EOF) { + return + } + api.Logger.Debug(r.Context(), "drpc server error", slog.Error(err)) + }, + }) + err = server.Serve(r.Context(), session) + if err != nil && !xerrors.Is(err, io.EOF) { + api.Logger.Debug(r.Context(), "provisioner daemon disconnected", slog.Error(err)) + _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("serve: %s", err)) + return + } + _ = conn.Close(websocket.StatusGoingAway, "") +} + +func (api *API) postProvisionerDaemon(rw http.ResponseWriter, r *http.Request) { + if !api.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { + httpapi.Forbidden(rw) + return + } + + var req codersdk.CreateProvisionerDaemonRequest + if !httpapi.Read(r.Context(), rw, r, &req) { + return + } + + provisioner, err := api.Database.InsertProvisionerDaemon(r.Context(), database.InsertProvisionerDaemonParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + Name: req.Name, + Provisioners: []database.ProvisionerType{database.ProvisionerTypeTerraform}, + AuthToken: uuid.NullUUID{Valid: true, UUID: uuid.New()}, + }) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Error inserting provisioner daemon.", + Detail: err.Error(), + }) + return + } + + httpapi.Write(r.Context(), rw, http.StatusCreated, convertProvisionerDaemon(provisioner)) +} + +// CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd. Useful when starting coderd and provisionerd // in the same process. -func (api *API) ListenProvisionerDaemon(ctx context.Context) (client proto.DRPCProvisionerDaemonClient, err error) { +func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client proto.DRPCProvisionerDaemonClient, err error) { clientSession, serverSession := provisionersdk.TransportPipe() defer func() { if err != nil { @@ -973,3 +1070,19 @@ func convertWorkspaceTransition(transition database.WorkspaceTransition) (sdkpro return 0, xerrors.Errorf("unrecognized transition: %q", transition) } } + +func convertProvisionerDaemon(daemon database.ProvisionerDaemon) codersdk.ProvisionerDaemon { + result := codersdk.ProvisionerDaemon{ + ID: daemon.ID, + CreatedAt: daemon.CreatedAt, + UpdatedAt: daemon.UpdatedAt, + Name: daemon.Name, + } + for _, provisionerType := range daemon.Provisioners { + result.Provisioners = append(result.Provisioners, codersdk.ProvisionerType(provisionerType)) + } + if daemon.AuthToken.Valid { + result.AuthToken = &daemon.AuthToken.UUID + } + return result +} diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index adce0321be691..0c05effdf1dc0 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -13,8 +13,12 @@ import ( "time" "github.com/google/uuid" + "github.com/hashicorp/yamux" "golang.org/x/xerrors" "nhooyr.io/websocket" + + "github.com/coder/coder/provisionerd/proto" + "github.com/coder/coder/provisionersdk" ) type LogSource string @@ -40,6 +44,7 @@ type ProvisionerDaemon struct { UpdatedAt sql.NullTime `json:"updated_at"` Name string `json:"name"` Provisioners []ProvisionerType `json:"provisioners"` + AuthToken *uuid.UUID `json:"auth_token"` } // ProvisionerJobStatus represents the at-time state of a job. @@ -162,3 +167,63 @@ func (c *Client) provisionerJobLogsAfter(ctx context.Context, path string, after return nil }), nil } + +type CreateProvisionerDaemonRequest struct { + Name string `json:"name" validate:"required"` +} + +// ListenProvisionerDaemon returns the gRPC service for a provisioner daemon implementation. +func (c *Client) ListenProvisionerDaemon(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) { + serverURL, err := c.URL.Parse("/api/v2/provisionerdaemons/me/listen") + if err != nil { + return nil, xerrors.Errorf("parse url: %w", err) + } + jar, err := cookiejar.New(nil) + if err != nil { + return nil, xerrors.Errorf("create cookie jar: %w", err) + } + jar.SetCookies(serverURL, []*http.Cookie{{ + Name: SessionTokenKey, + Value: c.SessionToken, + }}) + httpClient := &http.Client{ + Jar: jar, + } + conn, res, err := websocket.Dial(ctx, serverURL.String(), &websocket.DialOptions{ + HTTPClient: httpClient, + // Need to disable compression to avoid a data-race. + CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + if res == nil { + return nil, err + } + return nil, readBodyAsError(res) + } + // Align with the frame size of yamux. + conn.SetReadLimit(256 * 1024) + + config := yamux.DefaultConfig() + config.LogOutput = io.Discard + session, err := yamux.Client(websocket.NetConn(ctx, conn, websocket.MessageBinary), config) + if err != nil { + return nil, xerrors.Errorf("multiplex client: %w", err) + } + return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(session)), nil +} + +// CreateProvisionerDaemon creates a new standalone provisioner instance and generates an auth token. +func (c *Client) CreateProvisionerDaemon(ctx context.Context, req CreateProvisionerDaemonRequest) (ProvisionerDaemon, error) { + res, err := c.Request(ctx, http.MethodPost, "/api/v2/provisionerdaemons/", req) + if err != nil { + return ProvisionerDaemon{}, xerrors.Errorf("execute request: %w", err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusCreated { + return ProvisionerDaemon{}, readBodyAsError(res) + } + + var provisionerDaemon ProvisionerDaemon + return provisionerDaemon, json.NewDecoder(res.Body).Decode(&provisionerDaemon) +} From 3022f7b18296cd1eb62d942c45e0497b50271c92 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 7 Nov 2022 21:42:25 +0000 Subject: [PATCH 02/18] Move to Enterprise --- coderd/provisionerdaemons.go | 113 -------------------- enterprise/coderd/provisionerdaemons.go | 132 ++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 113 deletions(-) create mode 100644 enterprise/coderd/provisionerdaemons.go diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go index a3cee382383fd..570f698bfa0ed 100644 --- a/coderd/provisionerdaemons.go +++ b/coderd/provisionerdaemons.go @@ -13,12 +13,10 @@ import ( "time" "github.com/google/uuid" - "github.com/hashicorp/yamux" "github.com/moby/moby/pkg/namesgenerator" "github.com/tabbed/pqtype" "golang.org/x/xerrors" protobuf "google.golang.org/protobuf/proto" - "nhooyr.io/websocket" "storj.io/drpc/drpcmux" "storj.io/drpc/drpcserver" @@ -26,7 +24,6 @@ import ( "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/httpapi" - "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/parameter" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/coderd/telemetry" @@ -65,100 +62,6 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, daemons) } -// Serves the provisioner daemon protobuf API over a WebSocket. -func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request) { - daemon := httpmw.ProvisionerDaemon(r) - api.Logger.Warn(r.Context(), "daemon connected", slog.F("daemon", daemon.Name)) - - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() - - conn, err := websocket.Accept(rw, r, &websocket.AcceptOptions{ - // Need to disable compression to avoid a data-race. - CompressionMode: websocket.CompressionDisabled, - }) - if err != nil { - httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ - Message: "Internal error accepting websocket connection.", - Detail: err.Error(), - }) - return - } - // Align with the frame size of yamux. - conn.SetReadLimit(256 * 1024) - - // Multiplexes the incoming connection using yamux. - // This allows multiple function calls to occur over - // the same connection. - config := yamux.DefaultConfig() - config.LogOutput = io.Discard - session, err := yamux.Server(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), config) - if err != nil { - _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("multiplex server: %s", err)) - return - } - mux := drpcmux.New() - err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdServer{ - AccessURL: api.AccessURL, - ID: daemon.ID, - Database: api.Database, - Pubsub: api.Pubsub, - Provisioners: daemon.Provisioners, - Telemetry: api.Telemetry, - Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), - }) - if err != nil { - _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("drpc register provisioner daemon: %s", err)) - return - } - server := drpcserver.NewWithOptions(mux, drpcserver.Options{ - Log: func(err error) { - if xerrors.Is(err, io.EOF) { - return - } - api.Logger.Debug(r.Context(), "drpc server error", slog.Error(err)) - }, - }) - err = server.Serve(r.Context(), session) - if err != nil && !xerrors.Is(err, io.EOF) { - api.Logger.Debug(r.Context(), "provisioner daemon disconnected", slog.Error(err)) - _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("serve: %s", err)) - return - } - _ = conn.Close(websocket.StatusGoingAway, "") -} - -func (api *API) postProvisionerDaemon(rw http.ResponseWriter, r *http.Request) { - if !api.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { - httpapi.Forbidden(rw) - return - } - - var req codersdk.CreateProvisionerDaemonRequest - if !httpapi.Read(r.Context(), rw, r, &req) { - return - } - - provisioner, err := api.Database.InsertProvisionerDaemon(r.Context(), database.InsertProvisionerDaemonParams{ - ID: uuid.New(), - CreatedAt: database.Now(), - Name: req.Name, - Provisioners: []database.ProvisionerType{database.ProvisionerTypeTerraform}, - AuthToken: uuid.NullUUID{Valid: true, UUID: uuid.New()}, - }) - if err != nil { - httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Error inserting provisioner daemon.", - Detail: err.Error(), - }) - return - } - - httpapi.Write(r.Context(), rw, http.StatusCreated, convertProvisionerDaemon(provisioner)) -} - // CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd. Useful when starting coderd and provisionerd // in the same process. func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client proto.DRPCProvisionerDaemonClient, err error) { @@ -1106,19 +1009,3 @@ func convertWorkspaceTransition(transition database.WorkspaceTransition) (sdkpro return 0, xerrors.Errorf("unrecognized transition: %q", transition) } } - -func convertProvisionerDaemon(daemon database.ProvisionerDaemon) codersdk.ProvisionerDaemon { - result := codersdk.ProvisionerDaemon{ - ID: daemon.ID, - CreatedAt: daemon.CreatedAt, - UpdatedAt: daemon.UpdatedAt, - Name: daemon.Name, - } - for _, provisionerType := range daemon.Provisioners { - result.Provisioners = append(result.Provisioners, codersdk.ProvisionerType(provisionerType)) - } - if daemon.AuthToken.Valid { - result.AuthToken = &daemon.AuthToken.UUID - } - return result -} diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go new file mode 100644 index 0000000000000..e98e93896ea12 --- /dev/null +++ b/enterprise/coderd/provisionerdaemons.go @@ -0,0 +1,132 @@ +package coderd + +import ( + "fmt" + "io" + "net/http" + + "cdr.dev/slog" + "github.com/google/uuid" + "github.com/hashicorp/yamux" + "golang.org/x/xerrors" + "nhooyr.io/websocket" + "storj.io/drpc/drpcmux" + "storj.io/drpc/drpcserver" + + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/httpapi" + "github.com/coder/coder/coderd/httpmw" + "github.com/coder/coder/coderd/rbac" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/provisionerd/proto" +) + +func (api *API) postProvisionerDaemon(rw http.ResponseWriter, r *http.Request) { + if !api.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { + httpapi.Forbidden(rw) + return + } + + var req codersdk.CreateProvisionerDaemonRequest + if !httpapi.Read(r.Context(), rw, r, &req) { + return + } + + provisioner, err := api.Database.InsertProvisionerDaemon(r.Context(), database.InsertProvisionerDaemonParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + Name: req.Name, + Provisioners: []database.ProvisionerType{database.ProvisionerTypeTerraform}, + AuthToken: uuid.NullUUID{Valid: true, UUID: uuid.New()}, + }) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Error inserting provisioner daemon.", + Detail: err.Error(), + }) + return + } + + httpapi.Write(r.Context(), rw, http.StatusCreated, convertProvisionerDaemon(provisioner)) +} + +// Serves the provisioner daemon protobuf API over a WebSocket. +func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request) { + daemon := httpmw.ProvisionerDaemon(r) + api.Logger.Warn(r.Context(), "daemon connected", slog.F("daemon", daemon.Name)) + + api.websocketWaitMutex.Lock() + api.websocketWaitGroup.Add(1) + api.websocketWaitMutex.Unlock() + defer api.websocketWaitGroup.Done() + + conn, err := websocket.Accept(rw, r, &websocket.AcceptOptions{ + // Need to disable compression to avoid a data-race. + CompressionMode: websocket.CompressionDisabled, + }) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ + Message: "Internal error accepting websocket connection.", + Detail: err.Error(), + }) + return + } + // Align with the frame size of yamux. + conn.SetReadLimit(256 * 1024) + + // Multiplexes the incoming connection using yamux. + // This allows multiple function calls to occur over + // the same connection. + config := yamux.DefaultConfig() + config.LogOutput = io.Discard + session, err := yamux.Server(websocket.NetConn(r.Context(), conn, websocket.MessageBinary), config) + if err != nil { + _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("multiplex server: %s", err)) + return + } + mux := drpcmux.New() + err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdServer{ + AccessURL: api.AccessURL, + ID: daemon.ID, + Database: api.Database, + Pubsub: api.Pubsub, + Provisioners: daemon.Provisioners, + Telemetry: api.Telemetry, + Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), + }) + if err != nil { + _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("drpc register provisioner daemon: %s", err)) + return + } + server := drpcserver.NewWithOptions(mux, drpcserver.Options{ + Log: func(err error) { + if xerrors.Is(err, io.EOF) { + return + } + api.Logger.Debug(r.Context(), "drpc server error", slog.Error(err)) + }, + }) + err = server.Serve(r.Context(), session) + if err != nil && !xerrors.Is(err, io.EOF) { + api.Logger.Debug(r.Context(), "provisioner daemon disconnected", slog.Error(err)) + _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("serve: %s", err)) + return + } + _ = conn.Close(websocket.StatusGoingAway, "") +} + +func convertProvisionerDaemon(daemon database.ProvisionerDaemon) codersdk.ProvisionerDaemon { + result := codersdk.ProvisionerDaemon{ + ID: daemon.ID, + CreatedAt: daemon.CreatedAt, + UpdatedAt: daemon.UpdatedAt, + Name: daemon.Name, + } + for _, provisionerType := range daemon.Provisioners { + result.Provisioners = append(result.Provisioners, codersdk.ProvisionerType(provisionerType)) + } + if daemon.AuthToken.Valid { + result.AuthToken = &daemon.AuthToken.UUID + } + return result +} From 3f0738ecc3e2cbf90683f709df997325bbc0c065 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 8 Nov 2022 15:53:58 +0000 Subject: [PATCH 03/18] Begin adding tests for external registration --- coderd/coderd.go | 16 +++++----- coderd/provisionerjobs.go | 8 ++--- coderd/workspaceagents.go | 32 ++++++++++---------- enterprise/coderd/provisionerdaemons.go | 14 +++++---- enterprise/coderd/provisionerdaemons_test.go | 7 +++++ 5 files changed, 44 insertions(+), 33 deletions(-) create mode 100644 enterprise/coderd/provisionerdaemons_test.go diff --git a/coderd/coderd.go b/coderd/coderd.go index d81373638cd2d..a7bfe7d1f4b0e 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -599,18 +599,20 @@ type API struct { // RootHandler serves "/" RootHandler chi.Router - metricsCache *metricscache.Cache - siteHandler http.Handler - websocketWaitMutex sync.Mutex - websocketWaitGroup sync.WaitGroup + metricsCache *metricscache.Cache + siteHandler http.Handler + + WebsocketWaitMutex sync.Mutex + WebsocketWaitGroup sync.WaitGroup + workspaceAgentCache *wsconncache.Cache } // Close waits for all WebSocket connections to drain before returning. func (api *API) Close() error { - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Wait() - api.websocketWaitMutex.Unlock() + api.WebsocketWaitMutex.Lock() + api.WebsocketWaitGroup.Wait() + api.WebsocketWaitMutex.Unlock() api.metricsCache.Close() coordinator := api.TailnetCoordinator.Load() diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index dda3c2607fffa..8d72f09b581bd 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -130,10 +130,10 @@ func (api *API) provisionerJobLogs(rw http.ResponseWriter, r *http.Request, job return } - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() + api.WebsocketWaitMutex.Lock() + api.WebsocketWaitGroup.Add(1) + api.WebsocketWaitMutex.Unlock() + defer api.WebsocketWaitGroup.Done() conn, err := websocket.Accept(rw, r, nil) if err != nil { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ diff --git a/coderd/workspaceagents.go b/coderd/workspaceagents.go index 3a2ff65da1790..397e758efd151 100644 --- a/coderd/workspaceagents.go +++ b/coderd/workspaceagents.go @@ -181,10 +181,10 @@ func (api *API) postWorkspaceAgentVersion(rw http.ResponseWriter, r *http.Reques func (api *API) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() + api.WebsocketWaitMutex.Lock() + api.WebsocketWaitGroup.Add(1) + api.WebsocketWaitMutex.Unlock() + defer api.WebsocketWaitGroup.Done() workspaceAgent := httpmw.WorkspaceAgentParam(r) workspace := httpmw.WorkspaceParam(r) @@ -442,10 +442,10 @@ func (api *API) workspaceAgentConnection(rw http.ResponseWriter, r *http.Request func (api *API) workspaceAgentCoordinate(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() + api.WebsocketWaitMutex.Lock() + api.WebsocketWaitGroup.Add(1) + api.WebsocketWaitMutex.Unlock() + defer api.WebsocketWaitGroup.Done() workspaceAgent := httpmw.WorkspaceAgent(r) resource, err := api.Database.GetWorkspaceResourceByID(ctx, workspaceAgent.ResourceID) if err != nil { @@ -614,10 +614,10 @@ func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.R } } - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() + api.WebsocketWaitMutex.Lock() + api.WebsocketWaitGroup.Add(1) + api.WebsocketWaitMutex.Unlock() + defer api.WebsocketWaitGroup.Done() workspaceAgent := httpmw.WorkspaceAgentParam(r) conn, err := websocket.Accept(rw, r, nil) @@ -744,10 +744,10 @@ func convertWorkspaceAgent(derpMap *tailcfg.DERPMap, coordinator tailnet.Coordin func (api *API) workspaceAgentReportStats(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() + api.WebsocketWaitMutex.Lock() + api.WebsocketWaitGroup.Add(1) + api.WebsocketWaitMutex.Unlock() + defer api.WebsocketWaitGroup.Done() workspaceAgent := httpmw.WorkspaceAgent(r) resource, err := api.Database.GetWorkspaceResourceByID(ctx, workspaceAgent.ResourceID) diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index e98e93896ea12..ab650730cabd9 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -5,7 +5,6 @@ import ( "io" "net/http" - "cdr.dev/slog" "github.com/google/uuid" "github.com/hashicorp/yamux" "golang.org/x/xerrors" @@ -13,9 +12,12 @@ import ( "storj.io/drpc/drpcmux" "storj.io/drpc/drpcserver" + "cdr.dev/slog" + "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" + "github.com/coder/coder/coderd/provisionerdserver" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/codersdk" "github.com/coder/coder/provisionerd/proto" @@ -55,10 +57,10 @@ func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request daemon := httpmw.ProvisionerDaemon(r) api.Logger.Warn(r.Context(), "daemon connected", slog.F("daemon", daemon.Name)) - api.websocketWaitMutex.Lock() - api.websocketWaitGroup.Add(1) - api.websocketWaitMutex.Unlock() - defer api.websocketWaitGroup.Done() + api.AGPL.WebsocketWaitMutex.Lock() + api.AGPL.WebsocketWaitGroup.Add(1) + api.AGPL.WebsocketWaitMutex.Unlock() + defer api.AGPL.WebsocketWaitGroup.Done() conn, err := websocket.Accept(rw, r, &websocket.AcceptOptions{ // Need to disable compression to avoid a data-race. @@ -85,7 +87,7 @@ func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request return } mux := drpcmux.New() - err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdServer{ + err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdserver.Server{ AccessURL: api.AccessURL, ID: daemon.ID, Database: api.Database, diff --git a/enterprise/coderd/provisionerdaemons_test.go b/enterprise/coderd/provisionerdaemons_test.go new file mode 100644 index 0000000000000..7efa7839ae1ef --- /dev/null +++ b/enterprise/coderd/provisionerdaemons_test.go @@ -0,0 +1,7 @@ +package coderd_test + +import "testing" + +func TestPostProvisionerDaemon(t *testing.T) { + t.Parallel() +} From 0d29eaf162e7f05d7d8f1cae14453fe8f32a3bba Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 9 Nov 2022 15:03:25 +0000 Subject: [PATCH 04/18] Move provisioner daemons query to enterprise --- coderd/coderd.go | 70 +++++++++++++-- coderd/coderdtest/authorize.go | 16 ---- coderd/provisionerdaemons.go | 110 ------------------------ coderd/provisionerdaemons_test.go | 76 ---------------- enterprise/coderd/provisionerdaemons.go | 31 +++++++ 5 files changed, 94 insertions(+), 209 deletions(-) delete mode 100644 coderd/provisionerdaemons.go delete mode 100644 coderd/provisionerdaemons_test.go diff --git a/coderd/coderd.go b/coderd/coderd.go index a7bfe7d1f4b0e..81ec8bf653a99 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -1,6 +1,7 @@ package coderd import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -18,10 +19,13 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/google/uuid" "github.com/klauspost/compress/zstd" + "github.com/moby/moby/pkg/namesgenerator" "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/trace" "golang.org/x/xerrors" "google.golang.org/api/idtoken" + "storj.io/drpc/drpcmux" + "storj.io/drpc/drpcserver" "tailscale.com/derp" "tailscale.com/derp/derphttp" "tailscale.com/tailcfg" @@ -37,12 +41,15 @@ import ( "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" "github.com/coder/coder/coderd/metricscache" + "github.com/coder/coder/coderd/provisionerdserver" "github.com/coder/coder/coderd/rbac" "github.com/coder/coder/coderd/telemetry" "github.com/coder/coder/coderd/tracing" "github.com/coder/coder/coderd/workspacequota" "github.com/coder/coder/coderd/wsconncache" "github.com/coder/coder/codersdk" + "github.com/coder/coder/provisionerd/proto" + "github.com/coder/coder/provisionersdk" "github.com/coder/coder/site" "github.com/coder/coder/tailnet" ) @@ -329,13 +336,6 @@ func New(options *Options) *API { r.Get("/{fileID}", api.fileByID) r.Post("/", api.postFile) }) - - r.Route("/provisionerdaemons", func(r chi.Router) { - r.Use( - apiKeyMiddleware, - ) - r.Get("/", api.provisionerDaemons) - }) r.Route("/organizations", func(r chi.Router) { r.Use( apiKeyMiddleware, @@ -641,3 +641,59 @@ func compressHandler(h http.Handler) http.Handler { return cmp.Handler(h) } + +// CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd. Useful when starting coderd and provisionerd +// in the same process. +func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client proto.DRPCProvisionerDaemonClient, err error) { + clientSession, serverSession := provisionersdk.TransportPipe() + defer func() { + if err != nil { + _ = clientSession.Close() + _ = serverSession.Close() + } + }() + + name := namesgenerator.GetRandomName(1) + daemon, err := api.Database.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + Name: name, + Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, + }) + if err != nil { + return nil, xerrors.Errorf("insert provisioner daemon %q: %w", name, err) + } + + mux := drpcmux.New() + err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdserver.Server{ + AccessURL: api.AccessURL, + ID: daemon.ID, + Database: api.Database, + Pubsub: api.Pubsub, + Provisioners: daemon.Provisioners, + Telemetry: api.Telemetry, + Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), + }) + if err != nil { + return nil, err + } + server := drpcserver.NewWithOptions(mux, drpcserver.Options{ + Log: func(err error) { + if xerrors.Is(err, io.EOF) { + return + } + api.Logger.Debug(ctx, "drpc server error", slog.Error(err)) + }, + }) + go func() { + err := server.Serve(ctx, serverSession) + if err != nil && !xerrors.Is(err, io.EOF) { + api.Logger.Debug(ctx, "provisioner daemon disconnected", slog.Error(err)) + } + // close the sessions so we don't leak goroutines serving them. + _ = clientSession.Close() + _ = serverSession.Close() + }() + + return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(clientSession)), nil +} diff --git a/coderd/coderdtest/authorize.go b/coderd/coderdtest/authorize.go index e54b3104623ba..3af361808b1eb 100644 --- a/coderd/coderdtest/authorize.go +++ b/coderd/coderdtest/authorize.go @@ -19,7 +19,6 @@ import ( "github.com/coder/coder/codersdk" "github.com/coder/coder/provisioner/echo" "github.com/coder/coder/provisionersdk/proto" - "github.com/coder/coder/testutil" ) func AGPLRoutes(a *AuthTester) (map[string]string, map[string]RouteCheck) { @@ -204,11 +203,6 @@ func AGPLRoutes(a *AuthTester) (map[string]string, map[string]RouteCheck) { AssertAction: rbac.ActionRead, AssertObject: rbac.ResourceTemplate.InOrg(a.Version.OrganizationID), }, - "GET:/api/v2/provisionerdaemons": { - StatusCode: http.StatusOK, - AssertObject: rbac.ResourceProvisionerDaemon, - }, - "POST:/api/v2/parameters/{scope}/{id}": { AssertAction: rbac.ActionUpdate, AssertObject: rbac.ResourceTemplate, @@ -303,16 +297,6 @@ func NewAuthTester(ctx context.Context, t *testing.T, client *codersdk.Client, a if !ok { t.Fail() } - // The provisioner will call to coderd and register itself. This is async, - // so we wait for it to occur. - require.Eventually(t, func() bool { - provisionerds, err := client.ProvisionerDaemons(ctx) - return assert.NoError(t, err) && len(provisionerds) > 0 - }, testutil.WaitLong, testutil.IntervalSlow) - - provisionerds, err := client.ProvisionerDaemons(ctx) - require.NoError(t, err, "fetch provisioners") - require.Len(t, provisionerds, 1) organization, err := client.Organization(ctx, admin.OrganizationID) require.NoError(t, err, "fetch org") diff --git a/coderd/provisionerdaemons.go b/coderd/provisionerdaemons.go deleted file mode 100644 index b48ba5b4a750e..0000000000000 --- a/coderd/provisionerdaemons.go +++ /dev/null @@ -1,110 +0,0 @@ -package coderd - -import ( - "context" - "database/sql" - "errors" - "fmt" - "io" - "net/http" - - "github.com/google/uuid" - "github.com/moby/moby/pkg/namesgenerator" - "golang.org/x/xerrors" - "storj.io/drpc/drpcmux" - "storj.io/drpc/drpcserver" - - "cdr.dev/slog" - - "github.com/coder/coder/coderd/database" - "github.com/coder/coder/coderd/httpapi" - "github.com/coder/coder/coderd/provisionerdserver" - "github.com/coder/coder/coderd/rbac" - "github.com/coder/coder/codersdk" - "github.com/coder/coder/provisionerd/proto" - "github.com/coder/coder/provisionersdk" -) - -func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { - ctx := r.Context() - daemons, err := api.Database.GetProvisionerDaemons(ctx) - if errors.Is(err, sql.ErrNoRows) { - err = nil - } - if err != nil { - httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Internal error fetching provisioner daemons.", - Detail: err.Error(), - }) - return - } - if daemons == nil { - daemons = []database.ProvisionerDaemon{} - } - daemons, err = AuthorizeFilter(api.HTTPAuth, r, rbac.ActionRead, daemons) - if err != nil { - httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Internal error fetching provisioner daemons.", - Detail: err.Error(), - }) - return - } - - httpapi.Write(ctx, rw, http.StatusOK, daemons) -} - -// CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd. Useful when starting coderd and provisionerd -// in the same process. -func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client proto.DRPCProvisionerDaemonClient, err error) { - clientSession, serverSession := provisionersdk.TransportPipe() - defer func() { - if err != nil { - _ = clientSession.Close() - _ = serverSession.Close() - } - }() - - name := namesgenerator.GetRandomName(1) - daemon, err := api.Database.InsertProvisionerDaemon(ctx, database.InsertProvisionerDaemonParams{ - ID: uuid.New(), - CreatedAt: database.Now(), - Name: name, - Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, - }) - if err != nil { - return nil, xerrors.Errorf("insert provisioner daemon %q: %w", name, err) - } - - mux := drpcmux.New() - err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdserver.Server{ - AccessURL: api.AccessURL, - ID: daemon.ID, - Database: api.Database, - Pubsub: api.Pubsub, - Provisioners: daemon.Provisioners, - Telemetry: api.Telemetry, - Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), - }) - if err != nil { - return nil, err - } - server := drpcserver.NewWithOptions(mux, drpcserver.Options{ - Log: func(err error) { - if xerrors.Is(err, io.EOF) { - return - } - api.Logger.Debug(ctx, "drpc server error", slog.Error(err)) - }, - }) - go func() { - err := server.Serve(ctx, serverSession) - if err != nil && !xerrors.Is(err, io.EOF) { - api.Logger.Debug(ctx, "provisioner daemon disconnected", slog.Error(err)) - } - // close the sessions so we don't leak goroutines serving them. - _ = clientSession.Close() - _ = serverSession.Close() - }() - - return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(clientSession)), nil -} diff --git a/coderd/provisionerdaemons_test.go b/coderd/provisionerdaemons_test.go deleted file mode 100644 index d3b0be35cd020..0000000000000 --- a/coderd/provisionerdaemons_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package coderd_test - -import ( - "context" - "crypto/rand" - "runtime" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/coder/coder/coderd/coderdtest" - "github.com/coder/coder/codersdk" - "github.com/coder/coder/provisionersdk" - "github.com/coder/coder/testutil" -) - -func TestProvisionerDaemons(t *testing.T) { - t.Parallel() - t.Run("PayloadTooBig", func(t *testing.T) { - t.Parallel() - if runtime.GOOS == "windows" { - // Takes too long to allocate memory on Windows! - t.Skip() - } - client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true}) - user := coderdtest.CreateFirstUser(t, client) - data := make([]byte, provisionersdk.MaxMessageSize) - rand.Read(data) - - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() - - resp, err := client.Upload(ctx, codersdk.ContentTypeTar, data) - require.NoError(t, err) - t.Log(resp.ID) - - version, err := client.CreateTemplateVersion(ctx, user.OrganizationID, codersdk.CreateTemplateVersionRequest{ - StorageMethod: codersdk.ProvisionerStorageMethodFile, - FileID: resp.ID, - Provisioner: codersdk.ProvisionerTypeEcho, - }) - require.NoError(t, err) - require.Eventually(t, func() bool { - var err error - version, err = client.TemplateVersion(ctx, version.ID) - return assert.NoError(t, err) && version.Job.Error != "" - }, testutil.WaitShort, testutil.IntervalFast) - }) -} - -func TestProvisionerDaemonsByOrganization(t *testing.T) { - t.Parallel() - t.Run("NoAuth", func(t *testing.T) { - t.Parallel() - client := coderdtest.New(t, nil) - - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() - - _, err := client.ProvisionerDaemons(ctx) - require.Error(t, err) - }) - - t.Run("Get", func(t *testing.T) { - t.Parallel() - client := coderdtest.New(t, nil) - _ = coderdtest.CreateFirstUser(t, client) - - ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) - defer cancel() - - _, err := client.ProvisionerDaemons(ctx) - require.NoError(t, err) - }) -} diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index ab650730cabd9..57cc52b5f21f7 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -1,6 +1,8 @@ package coderd import ( + "database/sql" + "errors" "fmt" "io" "net/http" @@ -14,6 +16,7 @@ import ( "cdr.dev/slog" + "github.com/coder/coder/coderd" "github.com/coder/coder/coderd/database" "github.com/coder/coder/coderd/httpapi" "github.com/coder/coder/coderd/httpmw" @@ -23,6 +26,34 @@ import ( "github.com/coder/coder/provisionerd/proto" ) +func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { + ctx := r.Context() + daemons, err := api.Database.GetProvisionerDaemons(ctx) + if errors.Is(err, sql.ErrNoRows) { + err = nil + } + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error fetching provisioner daemons.", + Detail: err.Error(), + }) + return + } + if daemons == nil { + daemons = []database.ProvisionerDaemon{} + } + daemons, err = coderd.AuthorizeFilter(api.AGPL.HTTPAuth, r, rbac.ActionRead, daemons) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error fetching provisioner daemons.", + Detail: err.Error(), + }) + return + } + + httpapi.Write(ctx, rw, http.StatusOK, daemons) +} + func (api *API) postProvisionerDaemon(rw http.ResponseWriter, r *http.Request) { if !api.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { httpapi.Forbidden(rw) From 3033c58dd633609540aa0ea73925c5ce5ccf10a4 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 9 Nov 2022 20:00:19 +0000 Subject: [PATCH 05/18] Move around provisioner daemons schema --- coderd/database/dump.sql | 3 +- .../000065_provisioner_daemon_auth.down.sql | 1 - .../000065_provisioner_daemon_auth.up.sql | 1 - .../000072_provisioner_daemon_auth.down.sql | 2 ++ .../000072_provisioner_daemon_auth.up.sql | 3 ++ coderd/database/models.go | 15 +++++----- coderd/database/queries.sql.go | 12 +++++--- enterprise/coderd/coderd.go | 15 ++++++++-- enterprise/coderd/provisionerdaemons.go | 30 ------------------- 9 files changed, 35 insertions(+), 47 deletions(-) delete mode 100644 coderd/database/migrations/000065_provisioner_daemon_auth.down.sql delete mode 100644 coderd/database/migrations/000065_provisioner_daemon_auth.up.sql create mode 100644 coderd/database/migrations/000072_provisioner_daemon_auth.down.sql create mode 100644 coderd/database/migrations/000072_provisioner_daemon_auth.up.sql diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 02c2cdacdbda4..94a564d1c4b55 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -269,7 +269,8 @@ CREATE TABLE provisioner_daemons ( name character varying(64) NOT NULL, provisioners provisioner_type[] NOT NULL, replica_id uuid, - auth_token uuid + auth_token uuid, + tags jsonb ); CREATE TABLE provisioner_job_logs ( diff --git a/coderd/database/migrations/000065_provisioner_daemon_auth.down.sql b/coderd/database/migrations/000065_provisioner_daemon_auth.down.sql deleted file mode 100644 index fd5c4ad9ea6ee..0000000000000 --- a/coderd/database/migrations/000065_provisioner_daemon_auth.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE provisioner_daemons DROP COLUMN IF EXISTS auth_token; diff --git a/coderd/database/migrations/000065_provisioner_daemon_auth.up.sql b/coderd/database/migrations/000065_provisioner_daemon_auth.up.sql deleted file mode 100644 index 69c00e5375bfe..0000000000000 --- a/coderd/database/migrations/000065_provisioner_daemon_auth.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE provisioner_daemons ADD COLUMN IF NOT EXISTS auth_token uuid; diff --git a/coderd/database/migrations/000072_provisioner_daemon_auth.down.sql b/coderd/database/migrations/000072_provisioner_daemon_auth.down.sql new file mode 100644 index 0000000000000..f62cc99e6a266 --- /dev/null +++ b/coderd/database/migrations/000072_provisioner_daemon_auth.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE provisioner_daemons DROP COLUMN auth_token; +ALTER TABLE provisioner_daemons DROP COLUMN tags; diff --git a/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql b/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql new file mode 100644 index 0000000000000..8e56b9f8218d0 --- /dev/null +++ b/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE provisioner_daemons ADD COLUMN tags jsonb; + +ALTER TABLE template_versions ADD COLUMN provisioner_tags jsonb; diff --git a/coderd/database/models.go b/coderd/database/models.go index 116832e804356..f997c88da5166 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -518,13 +518,14 @@ type ParameterValue struct { } type ProvisionerDaemon struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt sql.NullTime `db:"updated_at" json:"updated_at"` - Name string `db:"name" json:"name"` - Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` - ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` - AuthToken uuid.NullUUID `db:"auth_token" json:"auth_token"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt sql.NullTime `db:"updated_at" json:"updated_at"` + Name string `db:"name" json:"name"` + Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` + ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` + AuthToken uuid.NullUUID `db:"auth_token" json:"auth_token"` + Tags pqtype.NullRawMessage `db:"tags" json:"tags"` } type ProvisionerJob struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index a654a2e4f147d..7cfed81ee2f78 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2226,7 +2226,7 @@ func (q *sqlQuerier) ParameterValues(ctx context.Context, arg ParameterValuesPar const getProvisionerDaemonByAuthToken = `-- name: GetProvisionerDaemonByAuthToken :one SELECT - id, created_at, updated_at, name, provisioners, replica_id, auth_token + id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags FROM provisioner_daemons WHERE @@ -2244,13 +2244,14 @@ func (q *sqlQuerier) GetProvisionerDaemonByAuthToken(ctx context.Context, authTo pq.Array(&i.Provisioners), &i.ReplicaID, &i.AuthToken, + &i.Tags, ) return i, err } const getProvisionerDaemonByID = `-- name: GetProvisionerDaemonByID :one SELECT - id, created_at, updated_at, name, provisioners, replica_id, auth_token + id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags FROM provisioner_daemons WHERE @@ -2268,13 +2269,14 @@ func (q *sqlQuerier) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) pq.Array(&i.Provisioners), &i.ReplicaID, &i.AuthToken, + &i.Tags, ) return i, err } const getProvisionerDaemons = `-- name: GetProvisionerDaemons :many SELECT - id, created_at, updated_at, name, provisioners, replica_id, auth_token + id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags FROM provisioner_daemons ` @@ -2296,6 +2298,7 @@ func (q *sqlQuerier) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDa pq.Array(&i.Provisioners), &i.ReplicaID, &i.AuthToken, + &i.Tags, ); err != nil { return nil, err } @@ -2320,7 +2323,7 @@ INSERT INTO auth_token ) VALUES - ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at, name, provisioners, replica_id, auth_token + ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags ` type InsertProvisionerDaemonParams struct { @@ -2348,6 +2351,7 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv pq.Array(&i.Provisioners), &i.ReplicaID, &i.AuthToken, + &i.Tags, ) return i, err } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index adebd113647b0..274272a801e69 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -90,7 +90,18 @@ func New(ctx context.Context, options *Options) (*API, error) { r.Get("/", api.group) }) }) - + r.Route("/organizations/{organization}/provisionerdaemons", func(r chi.Router) { + r.Use( + apiKeyMiddleware, + ) + r.Post("/", api.postProvisionerDaemonsByOrganization) + }) + r.Route("/provisionerdaemons", func(r chi.Router) { + r.Use(apiKeyMiddleware) + r.Get("/", api.provisionerDaemons) + r.Get("/listen", api.provisionerDaemonsListen) + r.Post("/", api.postProvisionerDaemonsByOrganization) + }) r.Route("/templates/{template}/acl", func(r chi.Router) { r.Use( api.templateRBACEnabledMW, @@ -100,7 +111,6 @@ func New(ctx context.Context, options *Options) (*API, error) { r.Get("/", api.templateACL) r.Patch("/", api.patchTemplateACL) }) - r.Route("/groups/{group}", func(r chi.Router) { r.Use( api.templateRBACEnabledMW, @@ -111,7 +121,6 @@ func New(ctx context.Context, options *Options) (*API, error) { r.Patch("/", api.patchGroup) r.Delete("/", api.deleteGroup) }) - r.Route("/workspace-quota", func(r chi.Router) { r.Use(apiKeyMiddleware) r.Route("/{user}", func(r chi.Router) { diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 57cc52b5f21f7..700c46d473f86 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -7,7 +7,6 @@ import ( "io" "net/http" - "github.com/google/uuid" "github.com/hashicorp/yamux" "golang.org/x/xerrors" "nhooyr.io/websocket" @@ -54,35 +53,6 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { httpapi.Write(ctx, rw, http.StatusOK, daemons) } -func (api *API) postProvisionerDaemon(rw http.ResponseWriter, r *http.Request) { - if !api.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { - httpapi.Forbidden(rw) - return - } - - var req codersdk.CreateProvisionerDaemonRequest - if !httpapi.Read(r.Context(), rw, r, &req) { - return - } - - provisioner, err := api.Database.InsertProvisionerDaemon(r.Context(), database.InsertProvisionerDaemonParams{ - ID: uuid.New(), - CreatedAt: database.Now(), - Name: req.Name, - Provisioners: []database.ProvisionerType{database.ProvisionerTypeTerraform}, - AuthToken: uuid.NullUUID{Valid: true, UUID: uuid.New()}, - }) - if err != nil { - httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Error inserting provisioner daemon.", - Detail: err.Error(), - }) - return - } - - httpapi.Write(r.Context(), rw, http.StatusCreated, convertProvisionerDaemon(provisioner)) -} - // Serves the provisioner daemon protobuf API over a WebSocket. func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request) { daemon := httpmw.ProvisionerDaemon(r) From 43268162c12ebeb3df17169ec4ce56e6ec76e8eb Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Thu, 10 Nov 2022 21:28:58 +0000 Subject: [PATCH 06/18] Add tags to provisioner daemons --- .vscode/settings.json | 1 + coderd/database/databasefake/databasefake.go | 26 ++----- coderd/database/dbtype/dbtype.go | 30 ++++++++ coderd/database/dump.sql | 6 +- .../000072_provisioner_daemon_auth.up.sql | 5 +- coderd/database/models.go | 17 ++--- coderd/database/querier.go | 1 - coderd/database/queries.sql.go | 66 +++++++----------- .../database/queries/provisionerdaemons.sql | 10 +-- coderd/database/queries/provisionerjobs.sql | 3 + coderd/database/sqlc.yaml | 4 ++ coderd/httpmw/provisionerdaemon.go | 69 ------------------- codersdk/provisionerdaemons.go | 36 ++++------ enterprise/coderd/coderd.go | 9 +-- enterprise/coderd/provisionerdaemons.go | 69 +++++++++++++++++-- enterprise/coderd/provisionerdaemons_test.go | 25 ++++++- site/src/api/typesGenerated.ts | 9 +++ 17 files changed, 194 insertions(+), 192 deletions(-) create mode 100644 coderd/database/dbtype/dbtype.go delete mode 100644 coderd/httpmw/provisionerdaemon.go diff --git a/.vscode/settings.json b/.vscode/settings.json index ba58f6f4ee1bf..09aab5fcbc198 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,7 @@ "codersdk", "cronstrue", "databasefake", + "dbtype", "DERP", "derphttp", "derpmap", diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index 34526b747cec6..dd874fd7a1470 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -3,6 +3,7 @@ package databasefake import ( "context" "database/sql" + "reflect" "sort" "strings" "sync" @@ -146,6 +147,9 @@ func (q *fakeQuerier) AcquireProvisionerJob(_ context.Context, arg database.Acqu if !found { continue } + if !reflect.DeepEqual(arg.Tags, provisionerJob.Tags) { + continue + } provisionerJob.StartedAt = arg.StartedAt provisionerJob.UpdatedAt = arg.StartedAt.Time provisionerJob.WorkerID = arg.WorkerID @@ -1890,26 +1894,6 @@ func (q *fakeQuerier) GetProvisionerDaemonByID(_ context.Context, id uuid.UUID) return database.ProvisionerDaemon{}, sql.ErrNoRows } -func (q *fakeQuerier) GetProvisionerDaemonByAuthToken(_ context.Context, token uuid.NullUUID) (database.ProvisionerDaemon, error) { - if !token.Valid { - return database.ProvisionerDaemon{}, sql.ErrNoRows - } - - q.mutex.RLock() - defer q.mutex.RUnlock() - - for _, provisionerDaemon := range q.provisionerDaemons { - if !provisionerDaemon.AuthToken.Valid { - continue - } - if provisionerDaemon.AuthToken.UUID.String() != token.UUID.String() { - continue - } - return provisionerDaemon, nil - } - return database.ProvisionerDaemon{}, sql.ErrNoRows -} - func (q *fakeQuerier) GetProvisionerJobByID(_ context.Context, id uuid.UUID) (database.ProvisionerJob, error) { q.mutex.RLock() defer q.mutex.RUnlock() @@ -2286,7 +2270,7 @@ func (q *fakeQuerier) InsertProvisionerDaemon(_ context.Context, arg database.In CreatedAt: arg.CreatedAt, Name: arg.Name, Provisioners: arg.Provisioners, - AuthToken: arg.AuthToken, + Tags: arg.Tags, } q.provisionerDaemons = append(q.provisionerDaemons, daemon) return daemon, nil diff --git a/coderd/database/dbtype/dbtype.go b/coderd/database/dbtype/dbtype.go new file mode 100644 index 0000000000000..40352897d0eee --- /dev/null +++ b/coderd/database/dbtype/dbtype.go @@ -0,0 +1,30 @@ +package dbtype + +import ( + "database/sql/driver" + "encoding/json" + + "golang.org/x/xerrors" +) + +type Map map[string]string + +func (m Map) Scan(src interface{}) error { + if src == nil { + return nil + } + switch src := src.(type) { + case []byte: + err := json.Unmarshal(src, &m) + if err != nil { + return err + } + default: + return xerrors.Errorf("unsupported Scan, storing driver.Value type %T into type %T", src, m) + } + return nil +} + +func (m Map) Value() (driver.Value, error) { + return json.Marshal(m) +} diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 94a564d1c4b55..dcb1e585eeb75 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -269,8 +269,7 @@ CREATE TABLE provisioner_daemons ( name character varying(64) NOT NULL, provisioners provisioner_type[] NOT NULL, replica_id uuid, - auth_token uuid, - tags jsonb + tags jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE provisioner_job_logs ( @@ -307,7 +306,8 @@ CREATE TABLE provisioner_jobs ( type provisioner_job_type NOT NULL, input jsonb NOT NULL, worker_id uuid, - file_id uuid NOT NULL + file_id uuid NOT NULL, + tags jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE replicas ( diff --git a/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql b/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql index 8e56b9f8218d0..164bae6fd4a50 100644 --- a/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql +++ b/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql @@ -1,3 +1,2 @@ -ALTER TABLE provisioner_daemons ADD COLUMN tags jsonb; - -ALTER TABLE template_versions ADD COLUMN provisioner_tags jsonb; +ALTER TABLE provisioner_daemons ADD COLUMN tags jsonb NOT NULL DEFAULT '{}'; +ALTER TABLE provisioner_jobs ADD COLUMN tags jsonb NOT NULL DEFAULT '{}'; diff --git a/coderd/database/models.go b/coderd/database/models.go index f997c88da5166..5316d464fe5b4 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -10,6 +10,7 @@ import ( "fmt" "time" + "github.com/coder/coder/coderd/database/dbtype" "github.com/google/uuid" "github.com/lib/pq" "github.com/tabbed/pqtype" @@ -518,14 +519,13 @@ type ParameterValue struct { } type ProvisionerDaemon struct { - ID uuid.UUID `db:"id" json:"id"` - CreatedAt time.Time `db:"created_at" json:"created_at"` - UpdatedAt sql.NullTime `db:"updated_at" json:"updated_at"` - Name string `db:"name" json:"name"` - Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` - ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` - AuthToken uuid.NullUUID `db:"auth_token" json:"auth_token"` - Tags pqtype.NullRawMessage `db:"tags" json:"tags"` + ID uuid.UUID `db:"id" json:"id"` + CreatedAt time.Time `db:"created_at" json:"created_at"` + UpdatedAt sql.NullTime `db:"updated_at" json:"updated_at"` + Name string `db:"name" json:"name"` + Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` + ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` + Tags dbtype.Map `db:"tags" json:"tags"` } type ProvisionerJob struct { @@ -544,6 +544,7 @@ type ProvisionerJob struct { Input json.RawMessage `db:"input" json:"input"` WorkerID uuid.NullUUID `db:"worker_id" json:"worker_id"` FileID uuid.UUID `db:"file_id" json:"file_id"` + Tags dbtype.Map `db:"tags" json:"tags"` } type ProvisionerJobLog struct { diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 1153bf1eecb3a..7a4fa43ffc5a5 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -65,7 +65,6 @@ type sqlcQuerier interface { GetParameterSchemasByJobID(ctx context.Context, jobID uuid.UUID) ([]ParameterSchema, error) GetParameterSchemasCreatedAfter(ctx context.Context, createdAt time.Time) ([]ParameterSchema, error) GetParameterValueByScopeAndName(ctx context.Context, arg GetParameterValueByScopeAndNameParams) (ParameterValue, error) - GetProvisionerDaemonByAuthToken(ctx context.Context, authToken uuid.NullUUID) (ProvisionerDaemon, error) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) (ProvisionerDaemon, error) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDaemon, error) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (ProvisionerJob, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 7cfed81ee2f78..a0ca7a03ac362 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -10,6 +10,7 @@ import ( "encoding/json" "time" + "github.com/coder/coder/coderd/database/dbtype" "github.com/google/uuid" "github.com/lib/pq" "github.com/tabbed/pqtype" @@ -2224,34 +2225,9 @@ func (q *sqlQuerier) ParameterValues(ctx context.Context, arg ParameterValuesPar return items, nil } -const getProvisionerDaemonByAuthToken = `-- name: GetProvisionerDaemonByAuthToken :one -SELECT - id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags -FROM - provisioner_daemons -WHERE - auth_token = $1 -` - -func (q *sqlQuerier) GetProvisionerDaemonByAuthToken(ctx context.Context, authToken uuid.NullUUID) (ProvisionerDaemon, error) { - row := q.db.QueryRowContext(ctx, getProvisionerDaemonByAuthToken, authToken) - var i ProvisionerDaemon - err := row.Scan( - &i.ID, - &i.CreatedAt, - &i.UpdatedAt, - &i.Name, - pq.Array(&i.Provisioners), - &i.ReplicaID, - &i.AuthToken, - &i.Tags, - ) - return i, err -} - const getProvisionerDaemonByID = `-- name: GetProvisionerDaemonByID :one SELECT - id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags + id, created_at, updated_at, name, provisioners, replica_id, tags FROM provisioner_daemons WHERE @@ -2268,7 +2244,6 @@ func (q *sqlQuerier) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) &i.Name, pq.Array(&i.Provisioners), &i.ReplicaID, - &i.AuthToken, &i.Tags, ) return i, err @@ -2276,7 +2251,7 @@ func (q *sqlQuerier) GetProvisionerDaemonByID(ctx context.Context, id uuid.UUID) const getProvisionerDaemons = `-- name: GetProvisionerDaemons :many SELECT - id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags + id, created_at, updated_at, name, provisioners, replica_id, tags FROM provisioner_daemons ` @@ -2297,7 +2272,6 @@ func (q *sqlQuerier) GetProvisionerDaemons(ctx context.Context) ([]ProvisionerDa &i.Name, pq.Array(&i.Provisioners), &i.ReplicaID, - &i.AuthToken, &i.Tags, ); err != nil { return nil, err @@ -2320,10 +2294,10 @@ INSERT INTO created_at, "name", provisioners, - auth_token + tags ) VALUES - ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at, name, provisioners, replica_id, auth_token, tags + ($1, $2, $3, $4, $5) RETURNING id, created_at, updated_at, name, provisioners, replica_id, tags ` type InsertProvisionerDaemonParams struct { @@ -2331,7 +2305,7 @@ type InsertProvisionerDaemonParams struct { CreatedAt time.Time `db:"created_at" json:"created_at"` Name string `db:"name" json:"name"` Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` - AuthToken uuid.NullUUID `db:"auth_token" json:"auth_token"` + Tags dbtype.Map `db:"tags" json:"tags"` } func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProvisionerDaemonParams) (ProvisionerDaemon, error) { @@ -2340,7 +2314,7 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv arg.CreatedAt, arg.Name, pq.Array(arg.Provisioners), - arg.AuthToken, + arg.Tags, ) var i ProvisionerDaemon err := row.Scan( @@ -2350,7 +2324,6 @@ func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProv &i.Name, pq.Array(&i.Provisioners), &i.ReplicaID, - &i.AuthToken, &i.Tags, ) return i, err @@ -2504,19 +2477,22 @@ WHERE AND nested.canceled_at IS NULL AND nested.completed_at IS NULL AND nested.provisioner = ANY($3 :: provisioner_type [ ]) + AND nested.tags @> $4 :: jsonb + AND nested.tags <@ $4 :: jsonb ORDER BY nested.created_at FOR UPDATE SKIP LOCKED LIMIT 1 - ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id + ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags ` type AcquireProvisionerJobParams struct { StartedAt sql.NullTime `db:"started_at" json:"started_at"` WorkerID uuid.NullUUID `db:"worker_id" json:"worker_id"` Types []ProvisionerType `db:"types" json:"types"` + Tags json.RawMessage `db:"tags" json:"tags"` } // Acquires the lock for a single job that isn't started, completed, @@ -2526,7 +2502,12 @@ type AcquireProvisionerJobParams struct { // multiple provisioners from acquiring the same jobs. See: // https://www.postgresql.org/docs/9.5/sql-select.html#SQL-FOR-UPDATE-SHARE func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvisionerJobParams) (ProvisionerJob, error) { - row := q.db.QueryRowContext(ctx, acquireProvisionerJob, arg.StartedAt, arg.WorkerID, pq.Array(arg.Types)) + row := q.db.QueryRowContext(ctx, acquireProvisionerJob, + arg.StartedAt, + arg.WorkerID, + pq.Array(arg.Types), + arg.Tags, + ) var i ProvisionerJob err := row.Scan( &i.ID, @@ -2544,13 +2525,14 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi &i.Input, &i.WorkerID, &i.FileID, + &i.Tags, ) return i, err } const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags FROM provisioner_jobs WHERE @@ -2576,13 +2558,14 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P &i.Input, &i.WorkerID, &i.FileID, + &i.Tags, ) return i, err } const getProvisionerJobsByIDs = `-- name: GetProvisionerJobsByIDs :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags FROM provisioner_jobs WHERE @@ -2614,6 +2597,7 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI &i.Input, &i.WorkerID, &i.FileID, + &i.Tags, ); err != nil { return nil, err } @@ -2629,7 +2613,7 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI } const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many -SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id FROM provisioner_jobs WHERE created_at > $1 +SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags FROM provisioner_jobs WHERE created_at > $1 ` func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) { @@ -2657,6 +2641,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created &i.Input, &i.WorkerID, &i.FileID, + &i.Tags, ); err != nil { return nil, err } @@ -2686,7 +2671,7 @@ INSERT INTO "input" ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags ` type InsertProvisionerJobParams struct { @@ -2732,6 +2717,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi &i.Input, &i.WorkerID, &i.FileID, + &i.Tags, ) return i, err } diff --git a/coderd/database/queries/provisionerdaemons.sql b/coderd/database/queries/provisionerdaemons.sql index e5f2e1b173a86..65908876e8a36 100644 --- a/coderd/database/queries/provisionerdaemons.sql +++ b/coderd/database/queries/provisionerdaemons.sql @@ -19,7 +19,7 @@ INSERT INTO created_at, "name", provisioners, - auth_token + tags ) VALUES ($1, $2, $3, $4, $5) RETURNING *; @@ -32,11 +32,3 @@ SET provisioners = $3 WHERE id = $1; - --- name: GetProvisionerDaemonByAuthToken :one -SELECT - * -FROM - provisioner_daemons -WHERE - auth_token = $1; diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index 027bd25bc91b3..605ee6a377464 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -22,6 +22,9 @@ WHERE AND nested.canceled_at IS NULL AND nested.completed_at IS NULL AND nested.provisioner = ANY(@types :: provisioner_type [ ]) + -- Ensure tags are equal! + AND nested.tags @> @tags :: jsonb + AND nested.tags <@ @tags :: jsonb ORDER BY nested.created_at FOR UPDATE diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 3a6d6e55b6853..bd96ad1bec2ea 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -17,6 +17,10 @@ packages: output_db_file_name: db_tmp.go overrides: + - column: "provisioner_daemons.tags" + go_type: "github.com/coder/coder/coderd/database/dbtype.Map" + - column: "provisioner_jobs.tags" + go_type: "github.com/coder/coder/coderd/database/dbtype.Map" - column: "users.rbac_roles" go_type: "github.com/lib/pq.StringArray" - column: "templates.user_acl" diff --git a/coderd/httpmw/provisionerdaemon.go b/coderd/httpmw/provisionerdaemon.go deleted file mode 100644 index 0d7f9c6f9d544..0000000000000 --- a/coderd/httpmw/provisionerdaemon.go +++ /dev/null @@ -1,69 +0,0 @@ -package httpmw - -import ( - "context" - "database/sql" - "errors" - "fmt" - "net/http" - - "github.com/google/uuid" - - "github.com/coder/coder/coderd/database" - "github.com/coder/coder/coderd/httpapi" - "github.com/coder/coder/codersdk" -) - -type provisionerDaemonContextKey struct{} - -// ProvisionerDaemon returns the daemon from the ExtractProvisionerDaemon handler. -func ProvisionerDaemon(r *http.Request) database.ProvisionerDaemon { - user, ok := r.Context().Value(provisionerDaemonContextKey{}).(database.ProvisionerDaemon) - if !ok { - panic("developer error: provisioner daemon middleware not provided") - } - return user -} - -// ExtractProvisionerDaemon requires authentication using a valid provisioner token. -func ExtractProvisionerDaemon(db database.Store) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { - cookie, err := r.Cookie(codersdk.SessionTokenKey) - if err != nil { - httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ - Message: fmt.Sprintf("Cookie %q must be provided.", codersdk.SessionTokenKey), - }) - return - } - token, err := uuid.Parse(cookie.Value) - if err != nil { - httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ - Message: "Provisioner token is invalid.", - }) - return - } - provisioner, err := db.GetProvisionerDaemonByAuthToken(r.Context(), uuid.NullUUID{ - Valid: true, - UUID: token, - }) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ - Message: "Provisioner token is invalid.", - }) - return - } - - httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ - Message: "Internal error fetching provisioner daemon.", - Detail: err.Error(), - }) - return - } - - ctx := context.WithValue(r.Context(), provisionerDaemonContextKey{}, provisioner) - next.ServeHTTP(rw, r.WithContext(ctx)) - }) - } -} diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index b44552d848cb2..6020a02cbfb2d 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -23,14 +23,12 @@ import ( type LogSource string +type LogLevel string + const ( LogSourceProvisionerDaemon LogSource = "provisioner_daemon" LogSourceProvisioner LogSource = "provisioner" -) - -type LogLevel string -const ( LogLevelTrace LogLevel = "trace" LogLevelDebug LogLevel = "debug" LogLevelInfo LogLevel = "info" @@ -44,7 +42,7 @@ type ProvisionerDaemon struct { UpdatedAt sql.NullTime `json:"updated_at"` Name string `json:"name"` Provisioners []ProvisionerType `json:"provisioners"` - AuthToken *uuid.UUID `json:"auth_token"` + Tags map[string]string `json:"tags"` } // ProvisionerJobStatus represents the at-time state of a job. @@ -173,11 +171,19 @@ type CreateProvisionerDaemonRequest struct { } // ListenProvisionerDaemon returns the gRPC service for a provisioner daemon implementation. -func (c *Client) ListenProvisionerDaemon(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) { - serverURL, err := c.URL.Parse("/api/v2/provisionerdaemons/me/listen") +func (c *Client) ServeProvisionerDaemon(ctx context.Context, organization uuid.UUID, provisioners []ProvisionerType, tags map[string]string) (proto.DRPCProvisionerDaemonClient, error) { + serverURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/organizations/%s/provisionerdaemons/serve", organization)) if err != nil { return nil, xerrors.Errorf("parse url: %w", err) } + query := serverURL.Query() + for _, provisioner := range provisioners { + query.Add("provisioner", string(provisioner)) + } + for key, value := range tags { + query.Add("tag", fmt.Sprintf("%s=%s", key, value)) + } + serverURL.RawQuery = query.Encode() jar, err := cookiejar.New(nil) if err != nil { return nil, xerrors.Errorf("create cookie jar: %w", err) @@ -211,19 +217,3 @@ func (c *Client) ListenProvisionerDaemon(ctx context.Context) (proto.DRPCProvisi } return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(session)), nil } - -// CreateProvisionerDaemon creates a new standalone provisioner instance and generates an auth token. -func (c *Client) CreateProvisionerDaemon(ctx context.Context, req CreateProvisionerDaemonRequest) (ProvisionerDaemon, error) { - res, err := c.Request(ctx, http.MethodPost, "/api/v2/provisionerdaemons/", req) - if err != nil { - return ProvisionerDaemon{}, xerrors.Errorf("execute request: %w", err) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusCreated { - return ProvisionerDaemon{}, readBodyAsError(res) - } - - var provisionerDaemon ProvisionerDaemon - return provisionerDaemon, json.NewDecoder(res.Body).Decode(&provisionerDaemon) -} diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 274272a801e69..f081bac26dda9 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -91,16 +91,9 @@ func New(ctx context.Context, options *Options) (*API, error) { }) }) r.Route("/organizations/{organization}/provisionerdaemons", func(r chi.Router) { - r.Use( - apiKeyMiddleware, - ) - r.Post("/", api.postProvisionerDaemonsByOrganization) - }) - r.Route("/provisionerdaemons", func(r chi.Router) { r.Use(apiKeyMiddleware) r.Get("/", api.provisionerDaemons) - r.Get("/listen", api.provisionerDaemonsListen) - r.Post("/", api.postProvisionerDaemonsByOrganization) + r.Get("/serve", api.provisionerDaemonServe) }) r.Route("/templates/{template}/acl", func(r chi.Router) { r.Use( diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 700c46d473f86..844b6597a6845 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -6,8 +6,11 @@ import ( "fmt" "io" "net/http" + "strings" + "github.com/google/uuid" "github.com/hashicorp/yamux" + "github.com/moby/moby/pkg/namesgenerator" "golang.org/x/xerrors" "nhooyr.io/websocket" "storj.io/drpc/drpcmux" @@ -54,9 +57,65 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { } // Serves the provisioner daemon protobuf API over a WebSocket. -func (api *API) provisionerDaemonsListen(rw http.ResponseWriter, r *http.Request) { - daemon := httpmw.ProvisionerDaemon(r) - api.Logger.Warn(r.Context(), "daemon connected", slog.F("daemon", daemon.Name)) +func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) { + tags := map[string]string{} + if r.URL.Query().Has("tag") { + for _, tag := range r.URL.Query()["tag"] { + parts := strings.SplitN(tag, "=", 2) + if len(parts) < 2 { + httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ + Message: fmt.Sprintf("Invalid format for tag %q. Key and value must be separated with =.", tag), + }) + return + } + tags[parts[0]] = parts[1] + } + } + if !r.URL.Query().Has("provisioner") { + httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ + Message: "The provisioner query parameter must be specified.", + }) + return + } + + provisioners := map[codersdk.ProvisionerType]struct{}{} + for _, provisioner := range r.URL.Query()["provisioner"] { + switch provisioner { + case string(codersdk.ProvisionerTypeEcho): + provisioners[codersdk.ProvisionerTypeEcho] = struct{}{} + case string(codersdk.ProvisionerTypeTerraform): + provisioners[codersdk.ProvisionerTypeTerraform] = struct{}{} + default: + httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ + Message: fmt.Sprintf("Unknown provisioner type %q", provisioner), + }) + return + } + } + + // Any authenticated user can create provisioner daemons scoped + // for jobs that they own, but only authorized users can create + // globally scoped provisioners that attach to all jobs. + apiKey := httpmw.APIKey(r) + if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { + tags["owner"] = apiKey.UserID.String() + } + + name := namesgenerator.GetRandomName(1) + daemon, err := api.Database.InsertProvisionerDaemon(r.Context(), database.InsertProvisionerDaemonParams{ + ID: uuid.New(), + CreatedAt: database.Now(), + Name: name, + Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, + Tags: tags, + }) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error writing provisioner daemon.", + Detail: err.Error(), + }) + return + } api.AGPL.WebsocketWaitMutex.Lock() api.AGPL.WebsocketWaitGroup.Add(1) @@ -124,12 +183,10 @@ func convertProvisionerDaemon(daemon database.ProvisionerDaemon) codersdk.Provis CreatedAt: daemon.CreatedAt, UpdatedAt: daemon.UpdatedAt, Name: daemon.Name, + Tags: daemon.Tags, } for _, provisionerType := range daemon.Provisioners { result.Provisioners = append(result.Provisioners, codersdk.ProvisionerType(provisionerType)) } - if daemon.AuthToken.Valid { - result.AuthToken = &daemon.AuthToken.UUID - } return result } diff --git a/enterprise/coderd/provisionerdaemons_test.go b/enterprise/coderd/provisionerdaemons_test.go index 7efa7839ae1ef..06c2b4148ff9f 100644 --- a/enterprise/coderd/provisionerdaemons_test.go +++ b/enterprise/coderd/provisionerdaemons_test.go @@ -1,6 +1,29 @@ package coderd_test -import "testing" +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/enterprise/coderd/coderdenttest" +) + +func TestProvisionerDaemonServe(t *testing.T) { + t.Parallel() + t.Run("Serve", func(t *testing.T) { + t.Parallel() + client := coderdenttest.New(t, nil) + user := coderdtest.CreateFirstUser(t, client) + srv, err := client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ + codersdk.ProvisionerTypeEcho, + }, map[string]string{}) + require.NoError(t, err) + srv.DRPCConn().Close() + }) +} func TestPostProvisionerDaemon(t *testing.T) { t.Parallel() diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 26a8e79115a2b..83746322db1a7 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -172,6 +172,11 @@ export interface CreateParameterRequest { readonly destination_scheme: ParameterDestinationScheme } +// From codersdk/provisionerdaemons.go +export interface CreateProvisionerDaemonRequest { + readonly name: string +} + // From codersdk/organizations.go export interface CreateTemplateRequest { readonly name: string @@ -521,6 +526,7 @@ export interface ProvisionerDaemon { readonly updated_at?: string readonly name: string readonly provisioners: ProvisionerType[] + readonly auth_token?: string } // From codersdk/provisionerdaemons.go @@ -957,6 +963,9 @@ export type ParameterSourceScheme = "data" | "none" // From codersdk/parameters.go export type ParameterTypeSystem = "hcl" | "none" +// From codersdk/provisionerdaemons.go +export type ProvisionerDaemonTag = "scope" + // From codersdk/provisionerdaemons.go export type ProvisionerJobStatus = | "canceled" From b1ce65b810f157220b96f16a4d6e043ccc9b00ca Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Sun, 13 Nov 2022 17:55:12 +0000 Subject: [PATCH 07/18] make gen --- ...wn.sql => 000072_provisioner_daemon_tags.down.sql} | 0 ...h.up.sql => 000072_provisioner_daemon_tags.up.sql} | 0 coderd/database/queries.sql.go | 11 +++++++---- coderd/database/queries/provisionerjobs.sql | 10 +++++----- 4 files changed, 12 insertions(+), 9 deletions(-) rename coderd/database/migrations/{000072_provisioner_daemon_auth.down.sql => 000072_provisioner_daemon_tags.down.sql} (100%) rename coderd/database/migrations/{000072_provisioner_daemon_auth.up.sql => 000072_provisioner_daemon_tags.up.sql} (100%) diff --git a/coderd/database/migrations/000072_provisioner_daemon_auth.down.sql b/coderd/database/migrations/000072_provisioner_daemon_tags.down.sql similarity index 100% rename from coderd/database/migrations/000072_provisioner_daemon_auth.down.sql rename to coderd/database/migrations/000072_provisioner_daemon_tags.down.sql diff --git a/coderd/database/migrations/000072_provisioner_daemon_auth.up.sql b/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql similarity index 100% rename from coderd/database/migrations/000072_provisioner_daemon_auth.up.sql rename to coderd/database/migrations/000072_provisioner_daemon_tags.up.sql diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index a0ca7a03ac362..e2876b09244f6 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2477,8 +2477,8 @@ WHERE AND nested.canceled_at IS NULL AND nested.completed_at IS NULL AND nested.provisioner = ANY($3 :: provisioner_type [ ]) - AND nested.tags @> $4 :: jsonb - AND nested.tags <@ $4 :: jsonb + -- Ensure the caller satisfies all job tags. + AND $4 :: jsonb @> nested.tags ORDER BY nested.created_at FOR UPDATE @@ -2668,10 +2668,11 @@ INSERT INTO storage_method, file_id, "type", - "input" + "input", + tags ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags ` type InsertProvisionerJobParams struct { @@ -2685,6 +2686,7 @@ type InsertProvisionerJobParams struct { FileID uuid.UUID `db:"file_id" json:"file_id"` Type ProvisionerJobType `db:"type" json:"type"` Input json.RawMessage `db:"input" json:"input"` + Tags dbtype.Map `db:"tags" json:"tags"` } func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) { @@ -2699,6 +2701,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi arg.FileID, arg.Type, arg.Input, + arg.Tags, ) var i ProvisionerJob err := row.Scan( diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index 605ee6a377464..17b51f15653d0 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -22,9 +22,8 @@ WHERE AND nested.canceled_at IS NULL AND nested.completed_at IS NULL AND nested.provisioner = ANY(@types :: provisioner_type [ ]) - -- Ensure tags are equal! - AND nested.tags @> @tags :: jsonb - AND nested.tags <@ @tags :: jsonb + -- Ensure the caller satisfies all job tags. + AND @tags :: jsonb @> nested.tags ORDER BY nested.created_at FOR UPDATE @@ -64,10 +63,11 @@ INSERT INTO storage_method, file_id, "type", - "input" + "input", + tags ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING *; -- name: UpdateProvisionerJobByID :exec UPDATE From e97287f8b33e6da5870ffd0cff6c0403bb8f3509 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 14 Nov 2022 21:57:50 +0000 Subject: [PATCH 08/18] Add user local provisioner daemons --- .../autobuild/executor/lifecycle_executor.go | 1 + coderd/coderd.go | 11 +++++++ coderd/database/databasefake/databasefake.go | 25 ++++++++++++-- coderd/database/dump.sql | 2 +- .../000072_provisioner_daemon_tags.up.sql | 5 ++- .../provisionerdserver/provisionerdserver.go | 2 ++ coderd/provisionerdserver/provisionertags.go | 33 +++++++++++++++++++ coderd/provisionerjobs.go | 1 + coderd/templateversions.go | 6 ++++ coderd/templateversions_test.go | 2 ++ coderd/workspacebuilds.go | 3 ++ coderd/workspaces.go | 3 ++ codersdk/organizations.go | 9 ++--- codersdk/provisionerdaemons.go | 5 +-- enterprise/coderd/coderd.go | 5 ++- .../coderdenttest/coderdenttest_test.go | 10 ++++++ enterprise/coderd/provisionerdaemons.go | 27 +++++++++++++-- site/src/api/typesGenerated.ts | 6 ++-- 18 files changed, 137 insertions(+), 19 deletions(-) create mode 100644 coderd/provisionerdserver/provisionertags.go diff --git a/coderd/autobuild/executor/lifecycle_executor.go b/coderd/autobuild/executor/lifecycle_executor.go index f21fde9a8af7b..ebe4857fb4070 100644 --- a/coderd/autobuild/executor/lifecycle_executor.go +++ b/coderd/autobuild/executor/lifecycle_executor.go @@ -277,6 +277,7 @@ func build(ctx context.Context, store database.Store, workspace database.Workspa Type: database.ProvisionerJobTypeWorkspaceBuild, StorageMethod: priorJob.StorageMethod, FileID: priorJob.FileID, + Tags: priorJob.Tags, Input: input, }) if err != nil { diff --git a/coderd/coderd.go b/coderd/coderd.go index 81ec8bf653a99..b5a09fcb6c1e0 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/json" "fmt" "io" "net/http" @@ -36,6 +37,7 @@ import ( "github.com/coder/coder/coderd/audit" "github.com/coder/coder/coderd/awsidentity" "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/database/dbtype" "github.com/coder/coder/coderd/gitauth" "github.com/coder/coder/coderd/gitsshkey" "github.com/coder/coder/coderd/httpapi" @@ -659,11 +661,19 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client pro CreatedAt: database.Now(), Name: name, Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, + Tags: dbtype.Map{ + provisionerdserver.TagScope: provisionerdserver.ScopeOrganization, + }, }) if err != nil { return nil, xerrors.Errorf("insert provisioner daemon %q: %w", name, err) } + tags, err := json.Marshal(daemon.Tags) + if err != nil { + return nil, xerrors.Errorf("marshal tags: %w", err) + } + mux := drpcmux.New() err = proto.DRPCRegisterProvisionerDaemon(mux, &provisionerdserver.Server{ AccessURL: api.AccessURL, @@ -672,6 +682,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context) (client pro Pubsub: api.Pubsub, Provisioners: daemon.Provisioners, Telemetry: api.Telemetry, + Tags: tags, Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), }) if err != nil { diff --git a/coderd/database/databasefake/databasefake.go b/coderd/database/databasefake/databasefake.go index dd874fd7a1470..f5a16999586d5 100644 --- a/coderd/database/databasefake/databasefake.go +++ b/coderd/database/databasefake/databasefake.go @@ -3,7 +3,7 @@ package databasefake import ( "context" "database/sql" - "reflect" + "encoding/json" "sort" "strings" "sync" @@ -147,7 +147,27 @@ func (q *fakeQuerier) AcquireProvisionerJob(_ context.Context, arg database.Acqu if !found { continue } - if !reflect.DeepEqual(arg.Tags, provisionerJob.Tags) { + tags := map[string]string{} + if arg.Tags != nil { + err := json.Unmarshal(arg.Tags, &tags) + if err != nil { + return provisionerJob, xerrors.Errorf("unmarshal: %w", err) + } + } + + missing := false + for key, value := range provisionerJob.Tags { + provided, found := tags[key] + if !found { + missing = true + break + } + if provided != value { + missing = true + break + } + } + if missing { continue } provisionerJob.StartedAt = arg.StartedAt @@ -2291,6 +2311,7 @@ func (q *fakeQuerier) InsertProvisionerJob(_ context.Context, arg database.Inser FileID: arg.FileID, Type: arg.Type, Input: arg.Input, + Tags: arg.Tags, } q.provisionerJobs = append(q.provisionerJobs, job) return job, nil diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index dcb1e585eeb75..f8ae502c19405 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -307,7 +307,7 @@ CREATE TABLE provisioner_jobs ( input jsonb NOT NULL, worker_id uuid, file_id uuid NOT NULL, - tags jsonb DEFAULT '{}'::jsonb NOT NULL + tags jsonb DEFAULT '{"scope": "organization"}'::jsonb NOT NULL ); CREATE TABLE replicas ( diff --git a/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql b/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql index 164bae6fd4a50..778214074625a 100644 --- a/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql +++ b/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql @@ -1,2 +1,5 @@ ALTER TABLE provisioner_daemons ADD COLUMN tags jsonb NOT NULL DEFAULT '{}'; -ALTER TABLE provisioner_jobs ADD COLUMN tags jsonb NOT NULL DEFAULT '{}'; + +-- We must add the organization scope by default, otherwise pending jobs +-- could be provisioned on new daemons that don't match the tags. +ALTER TABLE provisioner_jobs ADD COLUMN tags jsonb NOT NULL DEFAULT '{"scope":"organization"}'; diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 96415069e13a3..115023e9d8b92 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -32,6 +32,7 @@ type Server struct { ID uuid.UUID Logger slog.Logger Provisioners []database.ProvisionerType + Tags json.RawMessage Database database.Store Pubsub database.Pubsub Telemetry telemetry.Reporter @@ -50,6 +51,7 @@ func (server *Server) AcquireJob(ctx context.Context, _ *proto.Empty) (*proto.Ac Valid: true, }, Types: server.Provisioners, + Tags: server.Tags, }) if errors.Is(err, sql.ErrNoRows) { // The provisioner daemon assumes no jobs are available if diff --git a/coderd/provisionerdserver/provisionertags.go b/coderd/provisionerdserver/provisionertags.go new file mode 100644 index 0000000000000..7c9e029839d35 --- /dev/null +++ b/coderd/provisionerdserver/provisionertags.go @@ -0,0 +1,33 @@ +package provisionerdserver + +import "github.com/google/uuid" + +const ( + TagScope = "scope" + TagOwner = "owner" + + ScopeUser = "user" + ScopeOrganization = "organization" +) + +// MutateTags adjusts the "owner" tag dependent on the "scope". +// If the scope is "user", the "owner" is changed to the user ID. +// This is for user-scoped provisioner daemons, where users should +// own their own operations. +func MutateTags(userID uuid.UUID, tags map[string]string) map[string]string { + if tags == nil { + tags = map[string]string{} + } + _, ok := tags[TagScope] + if !ok { + tags[TagScope] = ScopeOrganization + } + switch tags[TagScope] { + case ScopeUser: + tags[TagOwner] = userID.String() + case ScopeOrganization: + default: + tags[TagScope] = ScopeOrganization + } + return tags +} diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index 8d72f09b581bd..972521612b90d 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -311,6 +311,7 @@ func convertProvisionerJob(provisionerJob database.ProvisionerJob) codersdk.Prov CreatedAt: provisionerJob.CreatedAt, Error: provisionerJob.Error.String, FileID: provisionerJob.FileID, + Tags: provisionerJob.Tags, } // Applying values optional to the struct. if provisionerJob.StartedAt.Valid { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 1d3ccb7919a23..7575fdc2651ab 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -288,6 +288,8 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques FileID: job.FileID, Type: database.ProvisionerJobTypeTemplateVersionDryRun, Input: input, + // Copy tags from the previous run. + Tags: job.Tags, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ @@ -717,6 +719,9 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht return } + // Ensures the "owner" is properly applied. + tags := provisionerdserver.MutateTags(apiKey.UserID, req.ProvisionerTags) + file, err := api.Database.GetFileByID(ctx, req.FileID) if errors.Is(err, sql.ErrNoRows) { httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{ @@ -815,6 +820,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht FileID: file.ID, Type: database.ProvisionerJobTypeTemplateVersionImport, Input: []byte{'{', '}'}, + Tags: tags, }) if err != nil { return xerrors.Errorf("insert provisioner job: %w", err) diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index 02ed6f1b9f30e..2f75c78fb77a2 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -13,6 +13,7 @@ import ( "github.com/coder/coder/coderd/audit" "github.com/coder/coder/coderd/coderdtest" "github.com/coder/coder/coderd/database" + "github.com/coder/coder/coderd/provisionerdserver" "github.com/coder/coder/codersdk" "github.com/coder/coder/provisioner/echo" "github.com/coder/coder/provisionersdk/proto" @@ -122,6 +123,7 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { }) require.NoError(t, err) require.Equal(t, "bananas", version.Name) + require.Equal(t, provisionerdserver.ScopeOrganization, version.Job.Tags[provisionerdserver.TagScope]) require.Len(t, auditor.AuditLogs, 1) assert.Equal(t, database.AuditActionCreate, auditor.AuditLogs[0].Action) diff --git a/coderd/workspacebuilds.go b/coderd/workspacebuilds.go index 54f8d2a9affaa..344ad39f796ae 100644 --- a/coderd/workspacebuilds.go +++ b/coderd/workspacebuilds.go @@ -428,6 +428,8 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) { return } + tags := provisionerdserver.MutateTags(workspace.OwnerID, templateVersionJob.Tags) + // Store prior build number to compute new build number var priorBuildNum int32 priorHistory, err := api.Database.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID) @@ -513,6 +515,7 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) { StorageMethod: templateVersionJob.StorageMethod, FileID: templateVersionJob.FileID, Input: input, + Tags: tags, }) if err != nil { return xerrors.Errorf("insert provisioner job: %w", err) diff --git a/coderd/workspaces.go b/coderd/workspaces.go index 2a18991e0fefc..b9cb01588ec56 100644 --- a/coderd/workspaces.go +++ b/coderd/workspaces.go @@ -428,6 +428,8 @@ func (api *API) postWorkspacesByOrganization(rw http.ResponseWriter, r *http.Req return } + tags := provisionerdserver.MutateTags(user.ID, templateVersionJob.Tags) + var ( provisionerJob database.ProvisionerJob workspaceBuild database.WorkspaceBuild @@ -490,6 +492,7 @@ func (api *API) postWorkspacesByOrganization(rw http.ResponseWriter, r *http.Req StorageMethod: templateVersionJob.StorageMethod, FileID: templateVersionJob.FileID, Input: input, + Tags: tags, }) if err != nil { return xerrors.Errorf("insert provisioner job: %w", err) diff --git a/codersdk/organizations.go b/codersdk/organizations.go index de5e42122ce28..deff2f8ba417b 100644 --- a/codersdk/organizations.go +++ b/codersdk/organizations.go @@ -36,11 +36,12 @@ type Organization struct { type CreateTemplateVersionRequest struct { Name string `json:"name,omitempty" validate:"omitempty,template_name"` // TemplateID optionally associates a version with a template. - TemplateID uuid.UUID `json:"template_id,omitempty"` + TemplateID uuid.UUID `json:"template_id,omitempty"` + StorageMethod ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"` + FileID uuid.UUID `json:"file_id" validate:"required"` + Provisioner ProvisionerType `json:"provisioner" validate:"oneof=terraform echo,required"` + ProvisionerTags map[string]string `json:"tags"` - StorageMethod ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"` - FileID uuid.UUID `json:"file_id" validate:"required"` - Provisioner ProvisionerType `json:"provisioner" validate:"oneof=terraform echo,required"` // ParameterValues allows for additional parameters to be provided // during the dry-run provision stage. ParameterValues []CreateParameterRequest `json:"parameter_values,omitempty"` diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index 6020a02cbfb2d..1ce19ffadba8d 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -76,6 +76,7 @@ type ProvisionerJob struct { Status ProvisionerJobStatus `json:"status"` WorkerID *uuid.UUID `json:"worker_id,omitempty"` FileID uuid.UUID `json:"file_id"` + Tags map[string]string `json:"tags"` } type ProvisionerJobLog struct { @@ -166,10 +167,6 @@ func (c *Client) provisionerJobLogsAfter(ctx context.Context, path string, after }), nil } -type CreateProvisionerDaemonRequest struct { - Name string `json:"name" validate:"required"` -} - // ListenProvisionerDaemon returns the gRPC service for a provisioner daemon implementation. func (c *Client) ServeProvisionerDaemon(ctx context.Context, organization uuid.UUID, provisioners []ProvisionerType, tags map[string]string) (proto.DRPCProvisionerDaemonClient, error) { serverURL, err := c.URL.Parse(fmt.Sprintf("/api/v2/organizations/%s/provisionerdaemons/serve", organization)) diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index f081bac26dda9..07ff8f0673529 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -91,7 +91,10 @@ func New(ctx context.Context, options *Options) (*API, error) { }) }) r.Route("/organizations/{organization}/provisionerdaemons", func(r chi.Router) { - r.Use(apiKeyMiddleware) + r.Use( + apiKeyMiddleware, + httpmw.ExtractOrganizationParam(api.Database), + ) r.Get("/", api.provisionerDaemons) r.Get("/serve", api.provisionerDaemonServe) }) diff --git a/enterprise/coderd/coderdenttest/coderdenttest_test.go b/enterprise/coderd/coderdenttest/coderdenttest_test.go index 319c805163271..3bc1fe237aa2a 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest_test.go +++ b/enterprise/coderd/coderdenttest/coderdenttest_test.go @@ -47,6 +47,8 @@ func TestAuthorizeAllEndpoints(t *testing.T) { a.URLParams["{groupName}"] = group.Name skipRoutes, assertRoute := coderdtest.AGPLRoutes(a) + skipRoutes["GET:/api/v2/organizations/{organization}/provisionerdaemons/serve"] = "This route checks for RBAC dependent on input parameters!" + assertRoute["GET:/api/v2/entitlements"] = coderdtest.RouteCheck{ NoAuthorize: true, } @@ -84,6 +86,14 @@ func TestAuthorizeAllEndpoints(t *testing.T) { AssertAction: rbac.ActionRead, AssertObject: groupObj, } + assertRoute["GET:/api/v2/organizations/{organization}/provisionerdaemons"] = coderdtest.RouteCheck{ + AssertAction: rbac.ActionRead, + AssertObject: rbac.ResourceProvisionerDaemon, + } + assertRoute["GET:/api/v2/organizations/{organization}/provisionerdaemons"] = coderdtest.RouteCheck{ + AssertAction: rbac.ActionRead, + AssertObject: rbac.ResourceProvisionerDaemon, + } assertRoute["GET:/api/v2/groups/{group}"] = coderdtest.RouteCheck{ AssertAction: rbac.ActionRead, AssertObject: groupObj, diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 844b6597a6845..8be98c4e03ba9 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -2,6 +2,7 @@ package coderd import ( "database/sql" + "encoding/json" "errors" "fmt" "io" @@ -30,6 +31,11 @@ import ( func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() + org := httpmw.OrganizationParam(r) + if !api.Authorize(r, rbac.ActionRead, rbac.ResourceProvisionerDaemon.InOrg(org.ID)) { + httpapi.Forbidden(rw) + return + } daemons, err := api.Database.GetProvisionerDaemons(ctx) if errors.Is(err, sql.ErrNoRows) { err = nil @@ -97,8 +103,15 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) // for jobs that they own, but only authorized users can create // globally scoped provisioners that attach to all jobs. apiKey := httpmw.APIKey(r) - if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { - tags["owner"] = apiKey.UserID.String() + tags = provisionerdserver.MutateTags(apiKey.UserID, tags) + + if tags[provisionerdserver.TagScope] == provisionerdserver.ScopeOrganization { + if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { + httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ + Message: "You aren't allowed to create provisioner daemons for the organization.", + }) + return + } } name := namesgenerator.GetRandomName(1) @@ -117,6 +130,15 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) return } + rawTags, err := json.Marshal(daemon.Tags) + if err != nil { + httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error marshaling daemon tags.", + Detail: err.Error(), + }) + return + } + api.AGPL.WebsocketWaitMutex.Lock() api.AGPL.WebsocketWaitGroup.Add(1) api.AGPL.WebsocketWaitMutex.Unlock() @@ -155,6 +177,7 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) Provisioners: daemon.Provisioners, Telemetry: api.Telemetry, Logger: api.Logger.Named(fmt.Sprintf("provisionerd-%s", daemon.Name)), + Tags: rawTags, }) if err != nil { _ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("drpc register provisioner daemon: %s", err)) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 83746322db1a7..4144068294636 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -201,6 +201,7 @@ export interface CreateTemplateVersionRequest { readonly storage_method: ProvisionerStorageMethod readonly file_id: string readonly provisioner: ProvisionerType + readonly tags: Record readonly parameter_values?: CreateParameterRequest[] } @@ -526,7 +527,7 @@ export interface ProvisionerDaemon { readonly updated_at?: string readonly name: string readonly provisioners: ProvisionerType[] - readonly auth_token?: string + readonly tags: Record } // From codersdk/provisionerdaemons.go @@ -963,9 +964,6 @@ export type ParameterSourceScheme = "data" | "none" // From codersdk/parameters.go export type ParameterTypeSystem = "hcl" | "none" -// From codersdk/provisionerdaemons.go -export type ProvisionerDaemonTag = "scope" - // From codersdk/provisionerdaemons.go export type ProvisionerJobStatus = | "canceled" From 5255b133356a6669ace3149fc01910bd60538275 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 15 Nov 2022 01:40:06 +0000 Subject: [PATCH 09/18] Add provisioner daemons --- enterprise/cli/provisionerdaemons.go | 1 + enterprise/coderd/provisionerdaemons.go | 4 +- enterprise/coderd/provisionerdaemons_test.go | 93 +++++++++++++++++++- provisionerd/provisionerd.go | 3 + 4 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 enterprise/cli/provisionerdaemons.go diff --git a/enterprise/cli/provisionerdaemons.go b/enterprise/cli/provisionerdaemons.go new file mode 100644 index 0000000000000..7f1e458cd3abe --- /dev/null +++ b/enterprise/cli/provisionerdaemons.go @@ -0,0 +1 @@ +package cli diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 8be98c4e03ba9..0f22207db760a 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -107,7 +107,7 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) if tags[provisionerdserver.TagScope] == provisionerdserver.ScopeOrganization { if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) { - httpapi.Write(r.Context(), rw, http.StatusUnauthorized, codersdk.Response{ + httpapi.Write(r.Context(), rw, http.StatusForbidden, codersdk.Response{ Message: "You aren't allowed to create provisioner daemons for the organization.", }) return @@ -130,6 +130,8 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) return } + fmt.Printf("TAGS %+v\n", daemon.Tags) + rawTags, err := json.Marshal(daemon.Tags) if err != nil { httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ diff --git a/enterprise/coderd/provisionerdaemons_test.go b/enterprise/coderd/provisionerdaemons_test.go index 06c2b4148ff9f..fdce821abee14 100644 --- a/enterprise/coderd/provisionerdaemons_test.go +++ b/enterprise/coderd/provisionerdaemons_test.go @@ -2,18 +2,26 @@ package coderd_test import ( "context" + "net/http" "testing" + "time" + "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/coder/coder/coderd/coderdtest" + "github.com/coder/coder/coderd/provisionerdserver" "github.com/coder/coder/codersdk" "github.com/coder/coder/enterprise/coderd/coderdenttest" + "github.com/coder/coder/provisioner/echo" + "github.com/coder/coder/provisionerd" + provisionerdproto "github.com/coder/coder/provisionerd/proto" + "github.com/coder/coder/provisionersdk/proto" ) func TestProvisionerDaemonServe(t *testing.T) { t.Parallel() - t.Run("Serve", func(t *testing.T) { + t.Run("Organization", func(t *testing.T) { t.Parallel() client := coderdenttest.New(t, nil) user := coderdtest.CreateFirstUser(t, client) @@ -23,6 +31,89 @@ func TestProvisionerDaemonServe(t *testing.T) { require.NoError(t, err) srv.DRPCConn().Close() }) + + t.Run("OrganizationNoPerms", func(t *testing.T) { + t.Parallel() + client := coderdenttest.New(t, nil) + user := coderdtest.CreateFirstUser(t, client) + another := coderdtest.CreateAnotherUser(t, client, user.OrganizationID) + _, err := another.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ + codersdk.ProvisionerTypeEcho, + }, map[string]string{ + provisionerdserver.TagScope: provisionerdserver.ScopeOrganization, + }) + require.Error(t, err) + var apiError *codersdk.Error + require.ErrorAs(t, err, &apiError) + require.Equal(t, http.StatusForbidden, apiError.StatusCode()) + }) + + t.Run("UserLocal", func(t *testing.T) { + t.Parallel() + client := coderdenttest.New(t, nil) + user := coderdtest.CreateFirstUser(t, client) + srv := provisionerd.New(func(ctx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) { + return client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ + codersdk.ProvisionerTypeEcho, + }, map[string]string{ + provisionerdserver.TagScope: provisionerdserver.ScopeUser, + }) + }, nil) + defer srv.Close() + + authToken := uuid.NewString() + data, err := echo.Tar(&echo.Responses{ + Parse: echo.ParseComplete, + ProvisionDryRun: []*proto.Provision_Response{{ + Type: &proto.Provision_Response_Complete{ + Complete: &proto.Provision_Complete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Name: "example", + }}, + }}, + }, + }, + }}, + Provision: []*proto.Provision_Response{{ + Type: &proto.Provision_Response_Complete{ + Complete: &proto.Provision_Complete{ + Resources: []*proto.Resource{{ + Name: "example", + Type: "aws_instance", + Agents: []*proto.Agent{{ + Id: uuid.NewString(), + Name: "example", + Auth: &proto.Agent_Token{ + Token: authToken, + }, + }}, + }}, + }, + }, + }}, + }) + require.NoError(t, err) + file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data) + require.NoError(t, err) + + _, err = client.CreateTemplateVersion(context.Background(), user.OrganizationID, codersdk.CreateTemplateVersionRequest{ + Name: "example", + StorageMethod: codersdk.ProvisionerStorageMethodFile, + FileID: file.ID, + Provisioner: codersdk.ProvisionerTypeEcho, + ProvisionerTags: map[string]string{ + provisionerdserver.TagScope: provisionerdserver.ScopeUser, + }, + }) + require.NoError(t, err) + // coderdtest.AwaitTemplateVersionJob(t, client, version.ID) + + time.Sleep(time.Second) + }) } func TestPostProvisionerDaemon(t *testing.T) { diff --git a/provisionerd/provisionerd.go b/provisionerd/provisionerd.go index f631f27208094..1a067306d2f49 100644 --- a/provisionerd/provisionerd.go +++ b/provisionerd/provisionerd.go @@ -58,6 +58,9 @@ type Options struct { // New creates and starts a provisioner daemon. func New(clientDialer Dialer, opts *Options) *Server { + if opts == nil { + opts = &Options{} + } if opts.PollInterval == 0 { opts.PollInterval = 5 * time.Second } From cf1221b1e8c2d111a44db89cf0fcbd213b067a56 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 15 Nov 2022 18:00:41 +0000 Subject: [PATCH 10/18] Add feature for external daemons --- coderd/coderdtest/coderdtest.go | 36 +++++++++++++ codersdk/features.go | 18 ++++--- .../coderd/coderdenttest/coderdenttest.go | 51 +++++++++++-------- enterprise/coderd/license/license.go | 23 ++++++--- enterprise/coderd/license/license_test.go | 51 ++++++++++--------- enterprise/coderd/provisionerdaemons.go | 16 ++++++ enterprise/coderd/provisionerdaemons_test.go | 30 +++++------ 7 files changed, 148 insertions(+), 77 deletions(-) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index a27ad9b95cf43..4624a5d2c9781 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -66,6 +66,7 @@ import ( "github.com/coder/coder/cryptorand" "github.com/coder/coder/provisioner/echo" "github.com/coder/coder/provisionerd" + provisionerdproto "github.com/coder/coder/provisionerd/proto" "github.com/coder/coder/provisionersdk" "github.com/coder/coder/provisionersdk/proto" "github.com/coder/coder/tailnet" @@ -337,6 +338,41 @@ func NewProvisionerDaemon(t *testing.T, coderAPI *coderd.API) io.Closer { return closer } +func NewExternalProvisionerDaemon(t *testing.T, client *codersdk.Client, org uuid.UUID, tags map[string]string) io.Closer { + echoClient, echoServer := provisionersdk.TransportPipe() + ctx, cancelFunc := context.WithCancel(context.Background()) + t.Cleanup(func() { + _ = echoClient.Close() + _ = echoServer.Close() + cancelFunc() + }) + fs := afero.NewMemMapFs() + go func() { + err := echo.Serve(ctx, fs, &provisionersdk.ServeOptions{ + Listener: echoServer, + }) + assert.NoError(t, err) + }() + + closer := provisionerd.New(func(ctx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) { + return client.ServeProvisionerDaemon(ctx, org, []codersdk.ProvisionerType{codersdk.ProvisionerTypeEcho}, tags) + }, &provisionerd.Options{ + Filesystem: fs, + Logger: slogtest.Make(t, nil).Named("provisionerd").Leveled(slog.LevelDebug), + PollInterval: 50 * time.Millisecond, + UpdateInterval: 250 * time.Millisecond, + ForceCancelInterval: time.Second, + Provisioners: provisionerd.Provisioners{ + string(database.ProvisionerTypeEcho): proto.NewDRPCProvisionerClient(provisionersdk.Conn(echoClient)), + }, + WorkDirectory: t.TempDir(), + }) + t.Cleanup(func() { + _ = closer.Close() + }) + return closer +} + var FirstUserParams = codersdk.CreateFirstUserRequest{ Email: "testuser@coder.com", Username: "testuser", diff --git a/codersdk/features.go b/codersdk/features.go index 336c01cc9eda9..5304b6fb2e377 100644 --- a/codersdk/features.go +++ b/codersdk/features.go @@ -15,14 +15,15 @@ const ( ) const ( - FeatureUserLimit = "user_limit" - FeatureAuditLog = "audit_log" - FeatureBrowserOnly = "browser_only" - FeatureSCIM = "scim" - FeatureWorkspaceQuota = "workspace_quota" - FeatureTemplateRBAC = "template_rbac" - FeatureHighAvailability = "high_availability" - FeatureMultipleGitAuth = "multiple_git_auth" + FeatureUserLimit = "user_limit" + FeatureAuditLog = "audit_log" + FeatureBrowserOnly = "browser_only" + FeatureSCIM = "scim" + FeatureWorkspaceQuota = "workspace_quota" + FeatureTemplateRBAC = "template_rbac" + FeatureHighAvailability = "high_availability" + FeatureMultipleGitAuth = "multiple_git_auth" + FeatureExternalProvisionerDaemons = "external_provisioner_daemons" ) var FeatureNames = []string{ @@ -34,6 +35,7 @@ var FeatureNames = []string{ FeatureTemplateRBAC, FeatureHighAvailability, FeatureMultipleGitAuth, + FeatureExternalProvisionerDaemons, } type Feature struct { diff --git a/enterprise/coderd/coderdenttest/coderdenttest.go b/enterprise/coderd/coderdenttest/coderdenttest.go index 341cb87dff1ec..8cd4905123f5e 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest.go +++ b/enterprise/coderd/coderdenttest/coderdenttest.go @@ -100,20 +100,21 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c } type LicenseOptions struct { - AccountType string - AccountID string - Trial bool - AllFeatures bool - GraceAt time.Time - ExpiresAt time.Time - UserLimit int64 - AuditLog bool - BrowserOnly bool - SCIM bool - WorkspaceQuota bool - TemplateRBAC bool - HighAvailability bool - MultipleGitAuth bool + AccountType string + AccountID string + Trial bool + AllFeatures bool + GraceAt time.Time + ExpiresAt time.Time + UserLimit int64 + AuditLog bool + BrowserOnly bool + SCIM bool + WorkspaceQuota bool + TemplateRBAC bool + HighAvailability bool + MultipleGitAuth bool + ExternalProvisionerDaemons bool } // AddLicense generates a new license with the options provided and inserts it. @@ -164,6 +165,11 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string { multipleGitAuth = 1 } + externalProvisionerDaemons := int64(0) + if options.ExternalProvisionerDaemons { + externalProvisionerDaemons = 1 + } + c := &license.Claims{ RegisteredClaims: jwt.RegisteredClaims{ Issuer: "test@testing.test", @@ -178,14 +184,15 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string { Version: license.CurrentVersion, AllFeatures: options.AllFeatures, Features: license.Features{ - UserLimit: options.UserLimit, - AuditLog: auditLog, - BrowserOnly: browserOnly, - SCIM: scim, - WorkspaceQuota: workspaceQuota, - HighAvailability: highAvailability, - TemplateRBAC: rbacEnabled, - MultipleGitAuth: multipleGitAuth, + UserLimit: options.UserLimit, + AuditLog: auditLog, + BrowserOnly: browserOnly, + SCIM: scim, + WorkspaceQuota: workspaceQuota, + HighAvailability: highAvailability, + TemplateRBAC: rbacEnabled, + MultipleGitAuth: multipleGitAuth, + ExternalProvisionerDaemons: externalProvisionerDaemons, }, } tok := jwt.NewWithClaims(jwt.SigningMethodEdDSA, c) diff --git a/enterprise/coderd/license/license.go b/enterprise/coderd/license/license.go index 80f502f527f2d..19d478f7ad4c9 100644 --- a/enterprise/coderd/license/license.go +++ b/enterprise/coderd/license/license.go @@ -123,6 +123,12 @@ func Entitlements( Enabled: true, } } + if claims.Features.ExternalProvisionerDaemons > 0 { + entitlements.Features[codersdk.FeatureExternalProvisionerDaemons] = codersdk.Feature{ + Entitlement: entitlement, + Enabled: true, + } + } if claims.AllFeatures { allFeatures = true } @@ -244,14 +250,15 @@ var ( ) type Features struct { - UserLimit int64 `json:"user_limit"` - AuditLog int64 `json:"audit_log"` - BrowserOnly int64 `json:"browser_only"` - SCIM int64 `json:"scim"` - WorkspaceQuota int64 `json:"workspace_quota"` - TemplateRBAC int64 `json:"template_rbac"` - HighAvailability int64 `json:"high_availability"` - MultipleGitAuth int64 `json:"multiple_git_auth"` + UserLimit int64 `json:"user_limit"` + AuditLog int64 `json:"audit_log"` + BrowserOnly int64 `json:"browser_only"` + SCIM int64 `json:"scim"` + WorkspaceQuota int64 `json:"workspace_quota"` + TemplateRBAC int64 `json:"template_rbac"` + HighAvailability int64 `json:"high_availability"` + MultipleGitAuth int64 `json:"multiple_git_auth"` + ExternalProvisionerDaemons int64 `json:"external_provisioner_daemons"` } type Claims struct { diff --git a/enterprise/coderd/license/license_test.go b/enterprise/coderd/license/license_test.go index a1b26f8dab9e4..e0bab820f6eca 100644 --- a/enterprise/coderd/license/license_test.go +++ b/enterprise/coderd/license/license_test.go @@ -20,13 +20,14 @@ import ( func TestEntitlements(t *testing.T) { t.Parallel() all := map[string]bool{ - codersdk.FeatureAuditLog: true, - codersdk.FeatureBrowserOnly: true, - codersdk.FeatureSCIM: true, - codersdk.FeatureWorkspaceQuota: true, - codersdk.FeatureHighAvailability: true, - codersdk.FeatureTemplateRBAC: true, - codersdk.FeatureMultipleGitAuth: true, + codersdk.FeatureAuditLog: true, + codersdk.FeatureBrowserOnly: true, + codersdk.FeatureSCIM: true, + codersdk.FeatureWorkspaceQuota: true, + codersdk.FeatureHighAvailability: true, + codersdk.FeatureTemplateRBAC: true, + codersdk.FeatureMultipleGitAuth: true, + codersdk.FeatureExternalProvisionerDaemons: true, } t.Run("Defaults", func(t *testing.T) { @@ -62,14 +63,15 @@ func TestEntitlements(t *testing.T) { db := databasefake.New() db.InsertLicense(context.Background(), database.InsertLicenseParams{ JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{ - UserLimit: 100, - AuditLog: true, - BrowserOnly: true, - SCIM: true, - WorkspaceQuota: true, - HighAvailability: true, - TemplateRBAC: true, - MultipleGitAuth: true, + UserLimit: 100, + AuditLog: true, + BrowserOnly: true, + SCIM: true, + WorkspaceQuota: true, + HighAvailability: true, + TemplateRBAC: true, + MultipleGitAuth: true, + ExternalProvisionerDaemons: true, }), Exp: time.Now().Add(time.Hour), }) @@ -86,15 +88,16 @@ func TestEntitlements(t *testing.T) { db := databasefake.New() db.InsertLicense(context.Background(), database.InsertLicenseParams{ JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{ - UserLimit: 100, - AuditLog: true, - BrowserOnly: true, - SCIM: true, - WorkspaceQuota: true, - HighAvailability: true, - TemplateRBAC: true, - GraceAt: time.Now().Add(-time.Hour), - ExpiresAt: time.Now().Add(time.Hour), + UserLimit: 100, + AuditLog: true, + BrowserOnly: true, + SCIM: true, + WorkspaceQuota: true, + HighAvailability: true, + TemplateRBAC: true, + ExternalProvisionerDaemons: true, + GraceAt: time.Now().Add(-time.Hour), + ExpiresAt: time.Now().Add(time.Hour), }), Exp: time.Now().Add(time.Hour), }) diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 0f22207db760a..3c1dc712c1322 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -29,6 +29,21 @@ import ( "github.com/coder/coder/provisionerd/proto" ) +func (api *API) provisionerDaemonsEnabledMW(next http.Handler) http.Handler { + return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + api.entitlementsMu.RLock() + epd := api.entitlements.Features[codersdk.FeatureExternalProvisionerDaemons].Enabled + api.entitlementsMu.RUnlock() + + if !epd { + httpapi.Write(r.Context(), rw, http.Status) + return + } + + next.ServeHTTP(rw, r) + }) +} + func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { ctx := r.Context() org := httpmw.OrganizationParam(r) @@ -64,6 +79,7 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { // Serves the provisioner daemon protobuf API over a WebSocket. func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) { + tags := map[string]string{} if r.URL.Query().Has("tag") { for _, tag := range r.URL.Query()["tag"] { diff --git a/enterprise/coderd/provisionerdaemons_test.go b/enterprise/coderd/provisionerdaemons_test.go index fdce821abee14..99d8bcfcac6dc 100644 --- a/enterprise/coderd/provisionerdaemons_test.go +++ b/enterprise/coderd/provisionerdaemons_test.go @@ -4,7 +4,6 @@ import ( "context" "net/http" "testing" - "time" "github.com/google/uuid" "github.com/stretchr/testify/require" @@ -14,8 +13,6 @@ import ( "github.com/coder/coder/codersdk" "github.com/coder/coder/enterprise/coderd/coderdenttest" "github.com/coder/coder/provisioner/echo" - "github.com/coder/coder/provisionerd" - provisionerdproto "github.com/coder/coder/provisionerd/proto" "github.com/coder/coder/provisionersdk/proto" ) @@ -52,14 +49,10 @@ func TestProvisionerDaemonServe(t *testing.T) { t.Parallel() client := coderdenttest.New(t, nil) user := coderdtest.CreateFirstUser(t, client) - srv := provisionerd.New(func(ctx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) { - return client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ - codersdk.ProvisionerTypeEcho, - }, map[string]string{ - provisionerdserver.TagScope: provisionerdserver.ScopeUser, - }) - }, nil) - defer srv.Close() + closer := coderdtest.NewExternalProvisionerDaemon(t, client, user.OrganizationID, map[string]string{ + provisionerdserver.TagScope: provisionerdserver.ScopeUser, + }) + defer closer.Close() authToken := uuid.NewString() data, err := echo.Tar(&echo.Responses{ @@ -100,7 +93,7 @@ func TestProvisionerDaemonServe(t *testing.T) { file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data) require.NoError(t, err) - _, err = client.CreateTemplateVersion(context.Background(), user.OrganizationID, codersdk.CreateTemplateVersionRequest{ + version, err := client.CreateTemplateVersion(context.Background(), user.OrganizationID, codersdk.CreateTemplateVersionRequest{ Name: "example", StorageMethod: codersdk.ProvisionerStorageMethodFile, FileID: file.ID, @@ -110,9 +103,16 @@ func TestProvisionerDaemonServe(t *testing.T) { }, }) require.NoError(t, err) - // coderdtest.AwaitTemplateVersionJob(t, client, version.ID) - - time.Sleep(time.Second) + coderdtest.AwaitTemplateVersionJob(t, client, version.ID) + template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID) + another := coderdtest.CreateAnotherUser(t, client, user.OrganizationID) + _ = closer.Close() + closer = coderdtest.NewExternalProvisionerDaemon(t, another, user.OrganizationID, map[string]string{ + provisionerdserver.TagScope: provisionerdserver.ScopeUser, + }) + defer closer.Close() + workspace := coderdtest.CreateWorkspace(t, another, user.OrganizationID, template.ID) + coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) }) } From 7dda3a2c59f63cc41142a53c10c27e27bb8639a2 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 15 Nov 2022 18:48:38 +0000 Subject: [PATCH 11/18] Add command to start a provisioner daemon --- cli/deployment/config.go | 4 +- cli/gitaskpass.go | 2 +- cli/gitssh.go | 2 +- cli/server.go | 2 +- cli/signal_unix.go | 2 +- cli/signal_windows.go | 2 +- enterprise/cli/provisionerdaemons.go | 157 +++++++++++++++++++ enterprise/cli/root.go | 1 + enterprise/coderd/coderd.go | 1 + enterprise/coderd/provisionerdaemons.go | 6 +- enterprise/coderd/provisionerdaemons_test.go | 26 ++- 11 files changed, 191 insertions(+), 14 deletions(-) diff --git a/cli/deployment/config.go b/cli/deployment/config.go index a77f687873e0f..6db10d77cc58b 100644 --- a/cli/deployment/config.go +++ b/cli/deployment/config.go @@ -143,7 +143,7 @@ func newConfig() *codersdk.DeploymentConfig { Name: "Cache Directory", Usage: "The directory to cache temporary files. If unspecified and $CACHE_DIRECTORY is set, it will be used for compatibility with systemd.", Flag: "cache-dir", - Default: defaultCacheDir(), + Default: DefaultCacheDir(), }, InMemoryDatabase: &codersdk.DeploymentConfigField[bool]{ Name: "In Memory Database", @@ -632,7 +632,7 @@ func formatEnv(key string) string { return "CODER_" + strings.ToUpper(strings.NewReplacer("-", "_", ".", "_").Replace(key)) } -func defaultCacheDir() string { +func DefaultCacheDir() string { defaultCacheDir, err := os.UserCacheDir() if err != nil { defaultCacheDir = os.TempDir() diff --git a/cli/gitaskpass.go b/cli/gitaskpass.go index 52af8b31da8a9..92723240c6b7e 100644 --- a/cli/gitaskpass.go +++ b/cli/gitaskpass.go @@ -26,7 +26,7 @@ func gitAskpass() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() - ctx, stop := signal.NotifyContext(ctx, interruptSignals...) + ctx, stop := signal.NotifyContext(ctx, InterruptSignals...) defer stop() user, host, err := gitauth.ParseAskpass(args[0]) diff --git a/cli/gitssh.go b/cli/gitssh.go index b18b919f79515..09ebc396fdbde 100644 --- a/cli/gitssh.go +++ b/cli/gitssh.go @@ -29,7 +29,7 @@ func gitssh() *cobra.Command { // Catch interrupt signals to ensure the temporary private // key file is cleaned up on most cases. - ctx, stop := signal.NotifyContext(ctx, interruptSignals...) + ctx, stop := signal.NotifyContext(ctx, InterruptSignals...) defer stop() // Early check so errors are reported immediately. diff --git a/cli/server.go b/cli/server.go index 1ed14232a7cb8..4b61d7cb3a3c0 100644 --- a/cli/server.go +++ b/cli/server.go @@ -104,7 +104,7 @@ func Server(vip *viper.Viper, newAPI func(context.Context, *coderd.Options) (*co // // To get out of a graceful shutdown, the user can send // SIGQUIT with ctrl+\ or SIGKILL with `kill -9`. - notifyCtx, notifyStop := signal.NotifyContext(ctx, interruptSignals...) + notifyCtx, notifyStop := signal.NotifyContext(ctx, InterruptSignals...) defer notifyStop() // Clean up idle connections at the end, e.g. diff --git a/cli/signal_unix.go b/cli/signal_unix.go index 7d2cd0e5022c5..05d619c0232e4 100644 --- a/cli/signal_unix.go +++ b/cli/signal_unix.go @@ -7,7 +7,7 @@ import ( "syscall" ) -var interruptSignals = []os.Signal{ +var InterruptSignals = []os.Signal{ os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, diff --git a/cli/signal_windows.go b/cli/signal_windows.go index 17652adfb626d..3624415a6452f 100644 --- a/cli/signal_windows.go +++ b/cli/signal_windows.go @@ -6,4 +6,4 @@ import ( "os" ) -var interruptSignals = []os.Signal{os.Interrupt} +var InterruptSignals = []os.Signal{os.Interrupt} diff --git a/enterprise/cli/provisionerdaemons.go b/enterprise/cli/provisionerdaemons.go index 7f1e458cd3abe..4198a76d73bdb 100644 --- a/enterprise/cli/provisionerdaemons.go +++ b/enterprise/cli/provisionerdaemons.go @@ -1 +1,158 @@ package cli + +import ( + "context" + "fmt" + "os" + "os/signal" + "strings" + "time" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/sloghuman" + agpl "github.com/coder/coder/cli" + "github.com/coder/coder/cli/cliflag" + "github.com/coder/coder/cli/cliui" + "github.com/coder/coder/cli/deployment" + "github.com/coder/coder/coderd/database" + "github.com/coder/coder/codersdk" + "github.com/coder/coder/provisioner/terraform" + "github.com/coder/coder/provisionerd" + provisionerdproto "github.com/coder/coder/provisionerd/proto" + "github.com/coder/coder/provisionersdk" + "github.com/coder/coder/provisionersdk/proto" + + "github.com/spf13/cobra" + "golang.org/x/xerrors" +) + +func provisionerDaemons() *cobra.Command { + cmd := &cobra.Command{ + Use: "provisionerd", + Short: "Manage provisioner daemons", + } + cmd.AddCommand(provisionerDaemonStart()) + + return cmd +} + +func provisionerDaemonStart() *cobra.Command { + var ( + cacheDir string + rawTags []string + ) + cmd := &cobra.Command{ + Use: "start", + Short: "Run a provisioner daemon", + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() + + notifyCtx, notifyStop := signal.NotifyContext(ctx, agpl.InterruptSignals...) + defer notifyStop() + + client, err := agpl.CreateClient(cmd) + if err != nil { + return xerrors.Errorf("create client: %w", err) + } + org, err := agpl.CurrentOrganization(cmd, client) + if err != nil { + return xerrors.Errorf("get current organization: %w", err) + } + + tags := map[string]string{} + for _, rawTag := range rawTags { + parts := strings.SplitN(rawTag, "=", 2) + if len(parts) < 2 { + return xerrors.Errorf("invalid tag format for %q. must be key=value", rawTag) + } + tags[parts[0]] = parts[1] + } + + err = os.MkdirAll(cacheDir, 0o700) + if err != nil { + return xerrors.Errorf("mkdir %q: %w", cacheDir, err) + } + + terraformClient, terraformServer := provisionersdk.TransportPipe() + go func() { + <-ctx.Done() + _ = terraformClient.Close() + _ = terraformServer.Close() + }() + + logger := slog.Make(sloghuman.Sink(cmd.ErrOrStderr())) + errCh := make(chan error, 1) + go func() { + defer cancel() + + err := terraform.Serve(ctx, &terraform.ServeOptions{ + ServeOptions: &provisionersdk.ServeOptions{ + Listener: terraformServer, + }, + CachePath: cacheDir, + Logger: logger.Named("terraform"), + }) + if err != nil && !xerrors.Is(err, context.Canceled) { + select { + case errCh <- err: + default: + } + } + }() + + tempDir, err := os.MkdirTemp("", "provisionerd") + if err != nil { + return err + } + + provisioners := provisionerd.Provisioners{ + string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(terraformClient)), + } + srv := provisionerd.New(func(ctx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) { + return client.ServeProvisionerDaemon(ctx, org.ID, []codersdk.ProvisionerType{ + codersdk.ProvisionerTypeTerraform, + }, tags) + }, &provisionerd.Options{ + Logger: logger, + PollInterval: 500 * time.Millisecond, + UpdateInterval: 500 * time.Millisecond, + Provisioners: provisioners, + WorkDirectory: tempDir, + }) + + var exitErr error + select { + case <-notifyCtx.Done(): + exitErr = notifyCtx.Err() + _, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Bold.Render( + "Interrupt caught, gracefully exiting. Use ctrl+\\ to force quit", + )) + case exitErr = <-errCh: + } + if exitErr != nil && !xerrors.Is(exitErr, context.Canceled) { + cmd.Printf("Unexpected error, shutting down server: %s\n", exitErr) + } + + shutdown, shutdownCancel := context.WithTimeout(ctx, time.Minute) + defer shutdownCancel() + err = srv.Shutdown(shutdown) + if err != nil { + return xerrors.Errorf("shutdown: %w", err) + } + + cancel() + if xerrors.Is(exitErr, context.Canceled) { + return nil + } + return exitErr + }, + } + + cliflag.StringVarP(cmd.Flags(), &cacheDir, "cache-dir", "c", "CODER_CACHE_DIRECTORY", deployment.DefaultCacheDir(), + "Specify a directory to cache provisioner job files.") + cliflag.StringArrayVarP(cmd.Flags(), &rawTags, "tag", "t", "CODER_PROVISIONERD_TAGS", []string{}, + "Specify a list of tags to target provisioner jobs.") + + return cmd +} diff --git a/enterprise/cli/root.go b/enterprise/cli/root.go index 41337f14c77dd..269bb7c0f1615 100644 --- a/enterprise/cli/root.go +++ b/enterprise/cli/root.go @@ -12,6 +12,7 @@ func enterpriseOnly() []*cobra.Command { features(), licenses(), groups(), + provisionerDaemons(), } } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 07ff8f0673529..5be0a1a722c4e 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -92,6 +92,7 @@ func New(ctx context.Context, options *Options) (*API, error) { }) r.Route("/organizations/{organization}/provisionerdaemons", func(r chi.Router) { r.Use( + api.provisionerDaemonsEnabledMW, apiKeyMiddleware, httpmw.ExtractOrganizationParam(api.Database), ) diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 3c1dc712c1322..6e5f2139d94d0 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -36,7 +36,9 @@ func (api *API) provisionerDaemonsEnabledMW(next http.Handler) http.Handler { api.entitlementsMu.RUnlock() if !epd { - httpapi.Write(r.Context(), rw, http.Status) + httpapi.Write(r.Context(), rw, http.StatusForbidden, codersdk.Response{ + Message: "External provisioner daemons is an Enterprise feature. Contact sales!", + }) return } @@ -146,8 +148,6 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) return } - fmt.Printf("TAGS %+v\n", daemon.Tags) - rawTags, err := json.Marshal(daemon.Tags) if err != nil { httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{ diff --git a/enterprise/coderd/provisionerdaemons_test.go b/enterprise/coderd/provisionerdaemons_test.go index 99d8bcfcac6dc..a7b9e125c4c9b 100644 --- a/enterprise/coderd/provisionerdaemons_test.go +++ b/enterprise/coderd/provisionerdaemons_test.go @@ -18,10 +18,26 @@ import ( func TestProvisionerDaemonServe(t *testing.T) { t.Parallel() + t.Run("NoLicense", func(t *testing.T) { + t.Parallel() + client := coderdenttest.New(t, nil) + user := coderdtest.CreateFirstUser(t, client) + _, err := client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ + codersdk.ProvisionerTypeEcho, + }, map[string]string{}) + require.Error(t, err) + var apiError *codersdk.Error + require.ErrorAs(t, err, &apiError) + require.Equal(t, http.StatusForbidden, apiError.StatusCode()) + }) + t.Run("Organization", func(t *testing.T) { t.Parallel() client := coderdenttest.New(t, nil) user := coderdtest.CreateFirstUser(t, client) + coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{ + ExternalProvisionerDaemons: true, + }) srv, err := client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ codersdk.ProvisionerTypeEcho, }, map[string]string{}) @@ -33,6 +49,9 @@ func TestProvisionerDaemonServe(t *testing.T) { t.Parallel() client := coderdenttest.New(t, nil) user := coderdtest.CreateFirstUser(t, client) + coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{ + ExternalProvisionerDaemons: true, + }) another := coderdtest.CreateAnotherUser(t, client, user.OrganizationID) _, err := another.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{ codersdk.ProvisionerTypeEcho, @@ -49,6 +68,9 @@ func TestProvisionerDaemonServe(t *testing.T) { t.Parallel() client := coderdenttest.New(t, nil) user := coderdtest.CreateFirstUser(t, client) + coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{ + ExternalProvisionerDaemons: true, + }) closer := coderdtest.NewExternalProvisionerDaemon(t, client, user.OrganizationID, map[string]string{ provisionerdserver.TagScope: provisionerdserver.ScopeUser, }) @@ -115,7 +137,3 @@ func TestProvisionerDaemonServe(t *testing.T) { coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID) }) } - -func TestPostProvisionerDaemon(t *testing.T) { - t.Parallel() -} From 200e6528ea24ff135a0807ef8efefa9b78c8287e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Tue, 15 Nov 2022 19:32:09 +0000 Subject: [PATCH 12/18] Add provisioner tags to template push and create --- cli/templatecreate.go | 32 +++++++++++++++++++++---- cli/templatepush.go | 17 +++++++++---- coderd/database/dbtype/dbtype.go | 4 ++-- enterprise/cli/provisionerdaemons.go | 13 ++++------ enterprise/coderd/coderd.go | 15 ++++++------ enterprise/coderd/provisionerdaemons.go | 26 ++++++++++++++------ 6 files changed, 73 insertions(+), 34 deletions(-) diff --git a/cli/templatecreate.go b/cli/templatecreate.go index 458cb1c2015b0..973054bba376a 100644 --- a/cli/templatecreate.go +++ b/cli/templatecreate.go @@ -27,6 +27,7 @@ func templateCreate() *cobra.Command { directory string provisioner string parameterFile string + provisionerTags []string maxTTL time.Duration minAutostartInterval time.Duration ) @@ -88,12 +89,18 @@ func templateCreate() *cobra.Command { } spin.Stop() + tags, err := ParseProvisionerTags(provisionerTags) + if err != nil { + return err + } + job, _, err := createValidTemplateVersion(cmd, createValidTemplateVersionArgs{ - Client: client, - Organization: organization, - Provisioner: database.ProvisionerType(provisioner), - FileID: resp.ID, - ParameterFile: parameterFile, + Client: client, + Organization: organization, + Provisioner: database.ProvisionerType(provisioner), + FileID: resp.ID, + ParameterFile: parameterFile, + ProvisionerTags: tags, }) if err != nil { return err @@ -135,6 +142,7 @@ func templateCreate() *cobra.Command { cmd.Flags().StringVarP(¶meterFile, "parameter-file", "", "", "Specify a file path with parameter values.") cmd.Flags().DurationVarP(&maxTTL, "max-ttl", "", 24*time.Hour, "Specify a maximum TTL for workspaces created from this template.") cmd.Flags().DurationVarP(&minAutostartInterval, "min-autostart-interval", "", time.Hour, "Specify a minimum autostart interval for workspaces created from this template.") + cmd.Flags().StringArrayVarP(&provisionerTags, "provisioner-tag", "", []string{}, "Specify a set of tags to target provisioner daemons.") // This is for testing! err := cmd.Flags().MarkHidden("test.provisioner") if err != nil { @@ -157,6 +165,7 @@ type createValidTemplateVersionArgs struct { // before prompting the user. Set to false to always prompt for param // values. ReuseParameters bool + ProvisionerTags map[string]string } func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVersionArgs, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) { @@ -168,6 +177,7 @@ func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVers FileID: args.FileID, Provisioner: codersdk.ProvisionerType(args.Provisioner), ParameterValues: parameters, + ProvisionerTags: args.ProvisionerTags, } if args.Template != nil { req.TemplateID = args.Template.ID @@ -337,3 +347,15 @@ func prettyDirectoryPath(dir string) string { } return pretty } + +func ParseProvisionerTags(rawTags []string) (map[string]string, error) { + tags := map[string]string{} + for _, rawTag := range rawTags { + parts := strings.SplitN(rawTag, "=", 2) + if len(parts) < 2 { + return nil, xerrors.Errorf("invalid tag format for %q. must be key=value", rawTag) + } + tags[parts[0]] = parts[1] + } + return tags, nil +} diff --git a/cli/templatepush.go b/cli/templatepush.go index 9eed180667e7c..c3a5e2f0c0ecb 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -18,11 +18,12 @@ import ( func templatePush() *cobra.Command { var ( - directory string - versionName string - provisioner string - parameterFile string - alwaysPrompt bool + directory string + versionName string + provisioner string + parameterFile string + alwaysPrompt bool + provisionerTags []string ) cmd := &cobra.Command{ @@ -75,6 +76,11 @@ func templatePush() *cobra.Command { } spin.Stop() + tags, err := ParseProvisionerTags(provisionerTags) + if err != nil { + return err + } + job, _, err := createValidTemplateVersion(cmd, createValidTemplateVersionArgs{ Name: versionName, Client: client, @@ -84,6 +90,7 @@ func templatePush() *cobra.Command { ParameterFile: parameterFile, Template: &template, ReuseParameters: !alwaysPrompt, + ProvisionerTags: tags, }) if err != nil { return err diff --git a/coderd/database/dbtype/dbtype.go b/coderd/database/dbtype/dbtype.go index 40352897d0eee..2d750a8f61df4 100644 --- a/coderd/database/dbtype/dbtype.go +++ b/coderd/database/dbtype/dbtype.go @@ -9,13 +9,13 @@ import ( type Map map[string]string -func (m Map) Scan(src interface{}) error { +func (m *Map) Scan(src interface{}) error { if src == nil { return nil } switch src := src.(type) { case []byte: - err := json.Unmarshal(src, &m) + err := json.Unmarshal(src, m) if err != nil { return err } diff --git a/enterprise/cli/provisionerdaemons.go b/enterprise/cli/provisionerdaemons.go index 4198a76d73bdb..285922d88bc06 100644 --- a/enterprise/cli/provisionerdaemons.go +++ b/enterprise/cli/provisionerdaemons.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "os/signal" - "strings" "time" "cdr.dev/slog" @@ -60,13 +59,9 @@ func provisionerDaemonStart() *cobra.Command { return xerrors.Errorf("get current organization: %w", err) } - tags := map[string]string{} - for _, rawTag := range rawTags { - parts := strings.SplitN(rawTag, "=", 2) - if len(parts) < 2 { - return xerrors.Errorf("invalid tag format for %q. must be key=value", rawTag) - } - tags[parts[0]] = parts[1] + tags, err := agpl.ParseProvisionerTags(rawTags) + if err != nil { + return err } err = os.MkdirAll(cacheDir, 0o700) @@ -106,6 +101,8 @@ func provisionerDaemonStart() *cobra.Command { return err } + logger.Info(ctx, "starting provisioner daemon", slog.F("tags", tags)) + provisioners := provisionerd.Provisioners{ string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(terraformClient)), } diff --git a/enterprise/coderd/coderd.go b/enterprise/coderd/coderd.go index 5be0a1a722c4e..fb626994daf99 100644 --- a/enterprise/coderd/coderd.go +++ b/enterprise/coderd/coderd.go @@ -227,13 +227,14 @@ func (api *API) updateEntitlements(ctx context.Context) error { defer api.entitlementsMu.Unlock() entitlements, err := license.Entitlements(ctx, api.Database, api.Logger, len(api.replicaManager.All()), len(api.GitAuthConfigs), api.Keys, map[string]bool{ - codersdk.FeatureAuditLog: api.AuditLogging, - codersdk.FeatureBrowserOnly: api.BrowserOnly, - codersdk.FeatureSCIM: len(api.SCIMAPIKey) != 0, - codersdk.FeatureWorkspaceQuota: api.UserWorkspaceQuota != 0, - codersdk.FeatureHighAvailability: api.DERPServerRelayAddress != "", - codersdk.FeatureMultipleGitAuth: len(api.GitAuthConfigs) > 1, - codersdk.FeatureTemplateRBAC: api.RBAC, + codersdk.FeatureAuditLog: api.AuditLogging, + codersdk.FeatureBrowserOnly: api.BrowserOnly, + codersdk.FeatureSCIM: len(api.SCIMAPIKey) != 0, + codersdk.FeatureWorkspaceQuota: api.UserWorkspaceQuota != 0, + codersdk.FeatureHighAvailability: api.DERPServerRelayAddress != "", + codersdk.FeatureMultipleGitAuth: len(api.GitAuthConfigs) > 1, + codersdk.FeatureTemplateRBAC: api.RBAC, + codersdk.FeatureExternalProvisionerDaemons: true, }) if err != nil { return err diff --git a/enterprise/coderd/provisionerdaemons.go b/enterprise/coderd/provisionerdaemons.go index 6e5f2139d94d0..e9f357f970e30 100644 --- a/enterprise/coderd/provisionerdaemons.go +++ b/enterprise/coderd/provisionerdaemons.go @@ -75,13 +75,15 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) { }) return } - - httpapi.Write(ctx, rw, http.StatusOK, daemons) + apiDaemons := make([]codersdk.ProvisionerDaemon, 0) + for _, daemon := range daemons { + apiDaemons = append(apiDaemons, convertProvisionerDaemon(daemon)) + } + httpapi.Write(ctx, rw, http.StatusOK, apiDaemons) } // Serves the provisioner daemon protobuf API over a WebSocket. func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) { - tags := map[string]string{} if r.URL.Query().Has("tag") { for _, tag := range r.URL.Query()["tag"] { @@ -102,13 +104,13 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) return } - provisioners := map[codersdk.ProvisionerType]struct{}{} + provisionersMap := map[codersdk.ProvisionerType]struct{}{} for _, provisioner := range r.URL.Query()["provisioner"] { switch provisioner { case string(codersdk.ProvisionerTypeEcho): - provisioners[codersdk.ProvisionerTypeEcho] = struct{}{} + provisionersMap[codersdk.ProvisionerTypeEcho] = struct{}{} case string(codersdk.ProvisionerTypeTerraform): - provisioners[codersdk.ProvisionerTypeTerraform] = struct{}{} + provisionersMap[codersdk.ProvisionerTypeTerraform] = struct{}{} default: httpapi.Write(r.Context(), rw, http.StatusBadRequest, codersdk.Response{ Message: fmt.Sprintf("Unknown provisioner type %q", provisioner), @@ -132,12 +134,22 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request) } } + provisioners := make([]database.ProvisionerType, 0) + for p := range provisionersMap { + switch p { + case codersdk.ProvisionerTypeTerraform: + provisioners = append(provisioners, database.ProvisionerTypeTerraform) + case codersdk.ProvisionerTypeEcho: + provisioners = append(provisioners, database.ProvisionerTypeEcho) + } + } + name := namesgenerator.GetRandomName(1) daemon, err := api.Database.InsertProvisionerDaemon(r.Context(), database.InsertProvisionerDaemonParams{ ID: uuid.New(), CreatedAt: database.Now(), Name: name, - Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, + Provisioners: provisioners, Tags: tags, }) if err != nil { From ac853f128d9d65b548afb62d4b74e1d8a2eea2a8 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 16 Nov 2022 15:26:45 +0000 Subject: [PATCH 13/18] Rename migration files --- ..._tags.down.sql => 000077_provisioner_daemon_tags.down.sql} | 0 ...emon_tags.up.sql => 000077_provisioner_daemon_tags.up.sql} | 0 enterprise/coderd/provisionerdaemons_test.go | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename coderd/database/migrations/{000072_provisioner_daemon_tags.down.sql => 000077_provisioner_daemon_tags.down.sql} (100%) rename coderd/database/migrations/{000072_provisioner_daemon_tags.up.sql => 000077_provisioner_daemon_tags.up.sql} (100%) diff --git a/coderd/database/migrations/000072_provisioner_daemon_tags.down.sql b/coderd/database/migrations/000077_provisioner_daemon_tags.down.sql similarity index 100% rename from coderd/database/migrations/000072_provisioner_daemon_tags.down.sql rename to coderd/database/migrations/000077_provisioner_daemon_tags.down.sql diff --git a/coderd/database/migrations/000072_provisioner_daemon_tags.up.sql b/coderd/database/migrations/000077_provisioner_daemon_tags.up.sql similarity index 100% rename from coderd/database/migrations/000072_provisioner_daemon_tags.up.sql rename to coderd/database/migrations/000077_provisioner_daemon_tags.up.sql diff --git a/enterprise/coderd/provisionerdaemons_test.go b/enterprise/coderd/provisionerdaemons_test.go index a7b9e125c4c9b..f603f3569e807 100644 --- a/enterprise/coderd/provisionerdaemons_test.go +++ b/enterprise/coderd/provisionerdaemons_test.go @@ -79,7 +79,7 @@ func TestProvisionerDaemonServe(t *testing.T) { authToken := uuid.NewString() data, err := echo.Tar(&echo.Responses{ Parse: echo.ParseComplete, - ProvisionDryRun: []*proto.Provision_Response{{ + ProvisionPlan: []*proto.Provision_Response{{ Type: &proto.Provision_Response_Complete{ Complete: &proto.Provision_Complete{ Resources: []*proto.Resource{{ @@ -93,7 +93,7 @@ func TestProvisionerDaemonServe(t *testing.T) { }, }, }}, - Provision: []*proto.Provision_Response{{ + ProvisionApply: []*proto.Provision_Response{{ Type: &proto.Provision_Response_Complete{ Complete: &proto.Provision_Complete{ Resources: []*proto.Resource{{ From d218655bdd576caccba57d1356396d415881ec2d Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 16 Nov 2022 15:37:36 +0000 Subject: [PATCH 14/18] Fix tests --- .../coderdenttest/coderdenttest_test.go | 3 +- enterprise/coderd/licenses_test.go | 30 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/enterprise/coderd/coderdenttest/coderdenttest_test.go b/enterprise/coderd/coderdenttest/coderdenttest_test.go index 3bc1fe237aa2a..e1a99291cd9f7 100644 --- a/enterprise/coderd/coderdenttest/coderdenttest_test.go +++ b/enterprise/coderd/coderdenttest/coderdenttest_test.go @@ -33,7 +33,8 @@ func TestAuthorizeAllEndpoints(t *testing.T) { ctx, _ := testutil.Context(t) admin := coderdtest.CreateFirstUser(t, client) license := coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{ - TemplateRBAC: true, + TemplateRBAC: true, + ExternalProvisionerDaemons: true, }) group, err := client.CreateGroup(ctx, admin.OrganizationID, codersdk.CreateGroupRequest{ Name: "testgroup", diff --git a/enterprise/coderd/licenses_test.go b/enterprise/coderd/licenses_test.go index 0605ff2f742c1..4e105b7831e08 100644 --- a/enterprise/coderd/licenses_test.go +++ b/enterprise/coderd/licenses_test.go @@ -101,25 +101,27 @@ func TestGetLicense(t *testing.T) { assert.Equal(t, int32(1), licenses[0].ID) assert.Equal(t, "testing", licenses[0].Claims["account_id"]) assert.Equal(t, map[string]interface{}{ - codersdk.FeatureUserLimit: json.Number("0"), - codersdk.FeatureAuditLog: json.Number("1"), - codersdk.FeatureSCIM: json.Number("1"), - codersdk.FeatureBrowserOnly: json.Number("1"), - codersdk.FeatureHighAvailability: json.Number("0"), - codersdk.FeatureTemplateRBAC: json.Number("1"), - codersdk.FeatureMultipleGitAuth: json.Number("0"), + codersdk.FeatureUserLimit: json.Number("0"), + codersdk.FeatureAuditLog: json.Number("1"), + codersdk.FeatureSCIM: json.Number("1"), + codersdk.FeatureBrowserOnly: json.Number("1"), + codersdk.FeatureHighAvailability: json.Number("0"), + codersdk.FeatureTemplateRBAC: json.Number("1"), + codersdk.FeatureMultipleGitAuth: json.Number("0"), + codersdk.FeatureExternalProvisionerDaemons: json.Number("0"), }, licenses[0].Claims["features"]) assert.Equal(t, int32(2), licenses[1].ID) assert.Equal(t, "testing2", licenses[1].Claims["account_id"]) assert.Equal(t, true, licenses[1].Claims["trial"]) assert.Equal(t, map[string]interface{}{ - codersdk.FeatureUserLimit: json.Number("200"), - codersdk.FeatureAuditLog: json.Number("1"), - codersdk.FeatureSCIM: json.Number("1"), - codersdk.FeatureBrowserOnly: json.Number("1"), - codersdk.FeatureHighAvailability: json.Number("0"), - codersdk.FeatureTemplateRBAC: json.Number("0"), - codersdk.FeatureMultipleGitAuth: json.Number("0"), + codersdk.FeatureUserLimit: json.Number("200"), + codersdk.FeatureAuditLog: json.Number("1"), + codersdk.FeatureSCIM: json.Number("1"), + codersdk.FeatureBrowserOnly: json.Number("1"), + codersdk.FeatureHighAvailability: json.Number("0"), + codersdk.FeatureTemplateRBAC: json.Number("0"), + codersdk.FeatureMultipleGitAuth: json.Number("0"), + codersdk.FeatureExternalProvisionerDaemons: json.Number("0"), }, licenses[1].Claims["features"]) }) } From b21004fd367511487c5a05536cc81523aeaa4a3e Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 16 Nov 2022 17:11:38 +0000 Subject: [PATCH 15/18] Fix entitlements test --- enterprise/coderd/coderd_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/enterprise/coderd/coderd_test.go b/enterprise/coderd/coderd_test.go index a6c3c6d86973b..6d8e90c9b185d 100644 --- a/enterprise/coderd/coderd_test.go +++ b/enterprise/coderd/coderd_test.go @@ -41,9 +41,10 @@ func TestEntitlements(t *testing.T) { }) _ = coderdtest.CreateFirstUser(t, client) coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{ - UserLimit: 100, - AuditLog: true, - TemplateRBAC: true, + UserLimit: 100, + AuditLog: true, + TemplateRBAC: true, + ExternalProvisionerDaemons: true, }) res, err := client.Entitlements(context.Background()) require.NoError(t, err) From ea94e1b82aa25f8418b84162dd81c8380c8b24a9 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 16 Nov 2022 21:40:57 +0000 Subject: [PATCH 16/18] PR comments --- coderd/coderd.go | 2 +- coderd/database/dbtype/dbtype.go | 6 +++--- .../migrations/000077_provisioner_daemon_tags.down.sql | 2 +- coderd/database/models.go | 4 ++-- coderd/database/queries.sql.go | 6 +++--- coderd/database/queries/provisionerjobs.sql | 2 +- coderd/database/sqlc.yaml | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/coderd/coderd.go b/coderd/coderd.go index 2619412516df9..077e8ecfc182c 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -656,7 +656,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, debounce ti CreatedAt: database.Now(), Name: name, Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform}, - Tags: dbtype.Map{ + Tags: dbtype.StringMap{ provisionerdserver.TagScope: provisionerdserver.ScopeOrganization, }, }) diff --git a/coderd/database/dbtype/dbtype.go b/coderd/database/dbtype/dbtype.go index 2d750a8f61df4..9ab47c16f5552 100644 --- a/coderd/database/dbtype/dbtype.go +++ b/coderd/database/dbtype/dbtype.go @@ -7,9 +7,9 @@ import ( "golang.org/x/xerrors" ) -type Map map[string]string +type StringMap map[string]string -func (m *Map) Scan(src interface{}) error { +func (m *StringMap) Scan(src interface{}) error { if src == nil { return nil } @@ -25,6 +25,6 @@ func (m *Map) Scan(src interface{}) error { return nil } -func (m Map) Value() (driver.Value, error) { +func (m StringMap) Value() (driver.Value, error) { return json.Marshal(m) } diff --git a/coderd/database/migrations/000077_provisioner_daemon_tags.down.sql b/coderd/database/migrations/000077_provisioner_daemon_tags.down.sql index f62cc99e6a266..4674e60ace5c2 100644 --- a/coderd/database/migrations/000077_provisioner_daemon_tags.down.sql +++ b/coderd/database/migrations/000077_provisioner_daemon_tags.down.sql @@ -1,2 +1,2 @@ -ALTER TABLE provisioner_daemons DROP COLUMN auth_token; ALTER TABLE provisioner_daemons DROP COLUMN tags; +ALTER TABLE provisioner_jobs DROP COLUMN tags; diff --git a/coderd/database/models.go b/coderd/database/models.go index 3a47f5f5c2c79..9b6f2cc1b163e 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -526,7 +526,7 @@ type ProvisionerDaemon struct { Name string `db:"name" json:"name"` Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` ReplicaID uuid.NullUUID `db:"replica_id" json:"replica_id"` - Tags dbtype.Map `db:"tags" json:"tags"` + Tags dbtype.StringMap `db:"tags" json:"tags"` } type ProvisionerJob struct { @@ -545,7 +545,7 @@ type ProvisionerJob struct { Input json.RawMessage `db:"input" json:"input"` WorkerID uuid.NullUUID `db:"worker_id" json:"worker_id"` FileID uuid.UUID `db:"file_id" json:"file_id"` - Tags dbtype.Map `db:"tags" json:"tags"` + Tags dbtype.StringMap `db:"tags" json:"tags"` } type ProvisionerJobLog struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index e986d09daab1d..db4dbb2f8a498 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -2322,7 +2322,7 @@ type InsertProvisionerDaemonParams struct { CreatedAt time.Time `db:"created_at" json:"created_at"` Name string `db:"name" json:"name"` Provisioners []ProvisionerType `db:"provisioners" json:"provisioners"` - Tags dbtype.Map `db:"tags" json:"tags"` + Tags dbtype.StringMap `db:"tags" json:"tags"` } func (q *sqlQuerier) InsertProvisionerDaemon(ctx context.Context, arg InsertProvisionerDaemonParams) (ProvisionerDaemon, error) { @@ -2495,7 +2495,7 @@ WHERE AND nested.completed_at IS NULL AND nested.provisioner = ANY($3 :: provisioner_type [ ]) -- Ensure the caller satisfies all job tags. - AND $4 :: jsonb @> nested.tags + AND nested.tags <@ $4 :: jsonb ORDER BY nested.created_at FOR UPDATE @@ -2703,7 +2703,7 @@ type InsertProvisionerJobParams struct { FileID uuid.UUID `db:"file_id" json:"file_id"` Type ProvisionerJobType `db:"type" json:"type"` Input json.RawMessage `db:"input" json:"input"` - Tags dbtype.Map `db:"tags" json:"tags"` + Tags dbtype.StringMap `db:"tags" json:"tags"` } func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) { diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index 17b51f15653d0..529c8d60e5608 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -23,7 +23,7 @@ WHERE AND nested.completed_at IS NULL AND nested.provisioner = ANY(@types :: provisioner_type [ ]) -- Ensure the caller satisfies all job tags. - AND @tags :: jsonb @> nested.tags + AND nested.tags <@ @tags :: jsonb ORDER BY nested.created_at FOR UPDATE diff --git a/coderd/database/sqlc.yaml b/coderd/database/sqlc.yaml index 33d4327c9fe4f..564a82eba1309 100644 --- a/coderd/database/sqlc.yaml +++ b/coderd/database/sqlc.yaml @@ -18,9 +18,9 @@ packages: overrides: - column: "provisioner_daemons.tags" - go_type: "github.com/coder/coder/coderd/database/dbtype.Map" + go_type: "github.com/coder/coder/coderd/database/dbtype.StringMap" - column: "provisioner_jobs.tags" - go_type: "github.com/coder/coder/coderd/database/dbtype.Map" + go_type: "github.com/coder/coder/coderd/database/dbtype.StringMap" - column: "users.rbac_roles" go_type: "github.com/lib/pq.StringArray" - column: "templates.user_acl" From e9b8e2db5c99e09ac704009fcfd076212ade19d4 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 16 Nov 2022 22:10:39 +0000 Subject: [PATCH 17/18] Update migration --- ...emon_tags.down.sql => 000079_provisioner_daemon_tags.down.sql} | 0 ...r_daemon_tags.up.sql => 000079_provisioner_daemon_tags.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000077_provisioner_daemon_tags.down.sql => 000079_provisioner_daemon_tags.down.sql} (100%) rename coderd/database/migrations/{000077_provisioner_daemon_tags.up.sql => 000079_provisioner_daemon_tags.up.sql} (100%) diff --git a/coderd/database/migrations/000077_provisioner_daemon_tags.down.sql b/coderd/database/migrations/000079_provisioner_daemon_tags.down.sql similarity index 100% rename from coderd/database/migrations/000077_provisioner_daemon_tags.down.sql rename to coderd/database/migrations/000079_provisioner_daemon_tags.down.sql diff --git a/coderd/database/migrations/000077_provisioner_daemon_tags.up.sql b/coderd/database/migrations/000079_provisioner_daemon_tags.up.sql similarity index 100% rename from coderd/database/migrations/000077_provisioner_daemon_tags.up.sql rename to coderd/database/migrations/000079_provisioner_daemon_tags.up.sql From d1f31f86922148eefc5ac83081a3e00266c2b7db Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Wed, 16 Nov 2022 22:19:37 +0000 Subject: [PATCH 18/18] Fix FE types --- site/src/api/api.ts | 2 ++ site/src/api/typesGenerated.ts | 6 +----- site/src/testHelpers/entities.ts | 2 ++ 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/site/src/api/api.ts b/site/src/api/api.ts index d3cf623453679..8b344af736936 100644 --- a/site/src/api/api.ts +++ b/site/src/api/api.ts @@ -69,12 +69,14 @@ export const provisioners: TypesGen.ProvisionerDaemon[] = [ name: "Terraform", created_at: "", provisioners: [], + tags: {}, }, { id: "cdr-basic", name: "Basic", created_at: "", provisioners: [], + tags: {}, }, ] diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 2e730a24cc8e5..7faaf7357c22c 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -173,11 +173,6 @@ export interface CreateParameterRequest { readonly destination_scheme: ParameterDestinationScheme } -// From codersdk/provisionerdaemons.go -export interface CreateProvisionerDaemonRequest { - readonly name: string -} - // From codersdk/organizations.go export interface CreateTemplateRequest { readonly name: string @@ -560,6 +555,7 @@ export interface ProvisionerJob { readonly status: ProvisionerJobStatus readonly worker_id?: string readonly file_id: string + readonly tags: Record } // From codersdk/provisionerdaemons.go diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index c43d7a64d19a8..7d1e5213518b7 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -131,6 +131,7 @@ export const MockProvisioner: TypesGen.ProvisionerDaemon = { id: "test-provisioner", name: "Test Provisioner", provisioners: ["echo"], + tags: {}, } export const MockProvisionerJob: TypesGen.ProvisionerJob = { @@ -139,6 +140,7 @@ export const MockProvisionerJob: TypesGen.ProvisionerJob = { status: "succeeded", file_id: "fc0774ce-cc9e-48d4-80ae-88f7a4d4a8b0", completed_at: "2022-05-17T17:39:01.382927298Z", + tags: {}, } export const MockFailedProvisionerJob: TypesGen.ProvisionerJob = {