Skip to content

Commit a6ce22d

Browse files
kylecarbskylecarbs
committed
Refactor provisionerd to dynamically update parameter schemas
1 parent 94eb484 commit a6ce22d

8 files changed

+390
-439
lines changed

‎coderd/provisionerdaemons.go

Lines changed: 90 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,6 @@ type workspaceProvisionJob struct {
113113
type projectVersionImportJob struct {
114114
OrganizationID string `json:"organization_id"`
115115
ProjectID uuid.UUID `json:"project_id"`
116-
117-
SkipParameterSchemas bool `json:"skip_parameter_schemas"`
118-
SkipResources bool `json:"skip_resources"`
119116
}
120117

121118
// Implementation of the provisioner daemon protobuf server.
@@ -276,9 +273,7 @@ func (server *provisionerdServer) AcquireJob(ctx context.Context, _ *proto.Empty
276273

277274
protoJob.Type = &proto.AcquiredJob_ProjectImport_{
278275
ProjectImport: &proto.AcquiredJob_ProjectImport{
279-
ParameterValues: protoParameters,
280-
SkipParameterSchemas: input.SkipParameterSchemas,
281-
SkipResources: input.SkipResources,
276+
ParameterValues: protoParameters,
282277
},
283278
}
284279
}
@@ -324,35 +319,89 @@ func (server *provisionerdServer) UpdateJob(stream proto.DRPCProvisionerDaemon_U
324319
if err != nil {
325320
return xerrors.Errorf("update job: %w", err)
326321
}
327-
insertParams := database.InsertProvisionerJobLogsParams{
328-
JobID: parsedID,
329-
}
330-
for _, log := range update.Logs {
331-
logLevel, err := convertLogLevel(log.Level)
322+
if len(update.Logs) > 0 {
323+
insertParams := database.InsertProvisionerJobLogsParams{
324+
JobID: parsedID,
325+
}
326+
for _, log := range update.Logs {
327+
logLevel, err := convertLogLevel(log.Level)
328+
if err != nil {
329+
return xerrors.Errorf("convert log level: %w", err)
330+
}
331+
logSource, err := convertLogSource(log.Source)
332+
if err != nil {
333+
return xerrors.Errorf("convert log source: %w", err)
334+
}
335+
insertParams.ID = append(insertParams.ID, uuid.New())
336+
insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt))
337+
insertParams.Level = append(insertParams.Level, logLevel)
338+
insertParams.Source = append(insertParams.Source, logSource)
339+
insertParams.Output = append(insertParams.Output, log.Output)
340+
}
341+
logs, err := server.Database.InsertProvisionerJobLogs(context.Background(), insertParams)
332342
if err != nil {
333-
return xerrors.Errorf("convert log level: %w", err)
343+
return xerrors.Errorf("insert job logs: %w", err)
334344
}
335-
logSource, err := convertLogSource(log.Source)
345+
data, err := json.Marshal(logs)
336346
if err != nil {
337-
return xerrors.Errorf("convert log source: %w", err)
347+
return xerrors.Errorf("marshal job log: %w", err)
348+
}
349+
err = server.Pubsub.Publish(provisionerJobLogsChannel(parsedID), data)
350+
if err != nil {
351+
return xerrors.Errorf("publish job log: %w", err)
338352
}
339-
insertParams.ID = append(insertParams.ID, uuid.New())
340-
insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt))
341-
insertParams.Level = append(insertParams.Level, logLevel)
342-
insertParams.Source = append(insertParams.Source, logSource)
343-
insertParams.Output = append(insertParams.Output, log.Output)
344-
}
345-
logs, err := server.Database.InsertProvisionerJobLogs(context.Background(), insertParams)
346-
if err != nil {
347-
return xerrors.Errorf("insert job logs: %w", err)
348-
}
349-
data, err := json.Marshal(logs)
350-
if err != nil {
351-
return xerrors.Errorf("marshal job log: %w", err)
352353
}
353-
err = server.Pubsub.Publish(provisionerJobLogsChannel(parsedID), data)
354-
if err != nil {
355-
return xerrors.Errorf("publish job log: %w", err)
354+
355+
if update.GetProjectImport() != nil {
356+
// Validate that all parameters send from the provisioner daemon
357+
// follow the protocol.
358+
parameterSchemas := make([]database.InsertParameterSchemaParams, 0, len(update.GetProjectImport().ParameterSchemas))
359+
for _, protoParameter := range update.GetProjectImport().ParameterSchemas {
360+
validationTypeSystem, err := convertValidationTypeSystem(protoParameter.ValidationTypeSystem)
361+
if err != nil {
362+
return xerrors.Errorf("convert validation type system for %q: %w", protoParameter.Name, err)
363+
}
364+
365+
parameterSchema := database.InsertParameterSchemaParams{
366+
ID: uuid.New(),
367+
CreatedAt: database.Now(),
368+
JobID: job.ID,
369+
Name: protoParameter.Name,
370+
Description: protoParameter.Description,
371+
RedisplayValue: protoParameter.RedisplayValue,
372+
ValidationError: protoParameter.ValidationError,
373+
ValidationCondition: protoParameter.ValidationCondition,
374+
ValidationValueType: protoParameter.ValidationValueType,
375+
ValidationTypeSystem: validationTypeSystem,
376+
377+
DefaultSourceScheme: database.ParameterSourceSchemeNone,
378+
DefaultDestinationScheme: database.ParameterDestinationSchemeNone,
379+
380+
AllowOverrideDestination: protoParameter.AllowOverrideDestination,
381+
AllowOverrideSource: protoParameter.AllowOverrideSource,
382+
}
383+
384+
// It's possible a parameter doesn't define a default source!
385+
if protoParameter.DefaultSource != nil {
386+
parameterSourceScheme, err := convertParameterSourceScheme(protoParameter.DefaultSource.Scheme)
387+
if err != nil {
388+
return xerrors.Errorf("convert parameter source scheme: %w", err)
389+
}
390+
parameterSchema.DefaultSourceScheme = parameterSourceScheme
391+
parameterSchema.DefaultSourceValue = protoParameter.DefaultSource.Value
392+
}
393+
394+
// It's possible a parameter doesn't define a default destination!
395+
if protoParameter.DefaultDestination != nil {
396+
parameterDestinationScheme, err := convertParameterDestinationScheme(protoParameter.DefaultDestination.Scheme)
397+
if err != nil {
398+
return xerrors.Errorf("convert parameter destination scheme: %w", err)
399+
}
400+
parameterSchema.DefaultDestinationScheme = parameterDestinationScheme
401+
}
402+
403+
parameterSchemas = append(parameterSchemas, parameterSchema)
404+
}
356405
}
357406
}
358407
}
@@ -412,80 +461,18 @@ func (server *provisionerdServer) CompleteJob(ctx context.Context, completed *pr
412461
return nil, xerrors.Errorf("unmarshal job data: %w", err)
413462
}
414463

415-
// Validate that all parameters send from the provisioner daemon
416-
// follow the protocol.
417-
parameterSchemas := make([]database.InsertParameterSchemaParams, 0, len(jobType.ProjectImport.ParameterSchemas))
418-
for _, protoParameter := range jobType.ProjectImport.ParameterSchemas {
419-
validationTypeSystem, err := convertValidationTypeSystem(protoParameter.ValidationTypeSystem)
420-
if err != nil {
421-
return nil, xerrors.Errorf("convert validation type system for %q: %w", protoParameter.Name, err)
422-
}
423-
424-
parameterSchema := database.InsertParameterSchemaParams{
425-
ID: uuid.New(),
426-
CreatedAt: database.Now(),
427-
JobID: job.ID,
428-
Name: protoParameter.Name,
429-
Description: protoParameter.Description,
430-
RedisplayValue: protoParameter.RedisplayValue,
431-
ValidationError: protoParameter.ValidationError,
432-
ValidationCondition: protoParameter.ValidationCondition,
433-
ValidationValueType: protoParameter.ValidationValueType,
434-
ValidationTypeSystem: validationTypeSystem,
435-
436-
DefaultSourceScheme: database.ParameterSourceSchemeNone,
437-
DefaultDestinationScheme: database.ParameterDestinationSchemeNone,
438-
439-
AllowOverrideDestination: protoParameter.AllowOverrideDestination,
440-
AllowOverrideSource: protoParameter.AllowOverrideSource,
441-
}
442-
443-
// It's possible a parameter doesn't define a default source!
444-
if protoParameter.DefaultSource != nil {
445-
parameterSourceScheme, err := convertParameterSourceScheme(protoParameter.DefaultSource.Scheme)
446-
if err != nil {
447-
return nil, xerrors.Errorf("convert parameter source scheme: %w", err)
448-
}
449-
parameterSchema.DefaultSourceScheme = parameterSourceScheme
450-
parameterSchema.DefaultSourceValue = protoParameter.DefaultSource.Value
451-
}
452-
453-
// It's possible a parameter doesn't define a default destination!
454-
if protoParameter.DefaultDestination != nil {
455-
parameterDestinationScheme, err := convertParameterDestinationScheme(protoParameter.DefaultDestination.Scheme)
456-
if err != nil {
457-
return nil, xerrors.Errorf("convert parameter destination scheme: %w", err)
458-
}
459-
parameterSchema.DefaultDestinationScheme = parameterDestinationScheme
460-
}
461-
462-
parameterSchemas = append(parameterSchemas, parameterSchema)
463-
}
464-
465-
// This must occur in a transaction in case of failure.
466-
err = server.Database.InTx(func(db database.Store) error {
467-
err = db.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{
468-
ID: jobID,
469-
UpdatedAt: database.Now(),
470-
CompletedAt: sql.NullTime{
471-
Time: database.Now(),
472-
Valid: true,
473-
},
474-
})
475-
if err != nil {
476-
return xerrors.Errorf("update provisioner job: %w", err)
477-
}
478-
// This could be a bulk-insert operation to improve performance.
479-
// See the "InsertWorkspaceHistoryLogs" query.
480-
for _, parameterSchema := range parameterSchemas {
481-
_, err = db.InsertParameterSchema(ctx, parameterSchema)
482-
if err != nil {
483-
return xerrors.Errorf("insert parameter schema %q: %w", parameterSchema.Name, err)
484-
}
485-
}
486-
server.Logger.Debug(ctx, "marked import job as completed", slog.F("job_id", jobID))
487-
return nil
464+
err = server.Database.UpdateProvisionerJobWithCompleteByID(ctx, database.UpdateProvisionerJobWithCompleteByIDParams{
465+
ID: jobID,
466+
UpdatedAt: database.Now(),
467+
CompletedAt: sql.NullTime{
468+
Time: database.Now(),
469+
Valid: true,
470+
},
488471
})
472+
if err != nil {
473+
return nil, xerrors.Errorf("update provisioner job: %w", err)
474+
}
475+
server.Logger.Debug(ctx, "marked import job as completed", slog.F("job_id", jobID))
489476
if err != nil {
490477
return nil, xerrors.Errorf("complete job: %w", err)
491478
}

‎coderd/provisionerjobs.go

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,10 @@ type ProvisionerJob struct {
4646
}
4747

4848
type CreateProjectImportJobRequest struct {
49-
StorageMethod database.ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"`
50-
StorageSource string `json:"storage_source" validate:"required"`
51-
Provisioner database.ProvisionerType `json:"provisioner" validate:"oneof=terraform echo,required"`
52-
53-
ParameterValues []CreateParameterValueRequest `json:"parameter_values"`
54-
SkipParameterSchemas bool `json:"skip_parameter_schemas"`
55-
SkipResources bool `json:"skip_resources"`
49+
StorageMethod database.ProvisionerStorageMethod `json:"storage_method" validate:"oneof=file,required"`
50+
StorageSource string `json:"storage_source" validate:"required"`
51+
Provisioner database.ProvisionerType `json:"provisioner" validate:"oneof=terraform echo,required"`
52+
ParameterValues []CreateParameterValueRequest `json:"parameter_values"`
5653
}
5754

5855
func (*api) provisionerJobByOrganization(rw http.ResponseWriter, r *http.Request) {
@@ -84,10 +81,7 @@ func (api *api) postProvisionerImportJobByOrganization(rw http.ResponseWriter, r
8481
}
8582

8683
input, err := json.Marshal(projectVersionImportJob{
87-
// AdditionalParameters: req.AdditionalParameters,
88-
OrganizationID: organization.ID,
89-
SkipParameterSchemas: req.SkipParameterSchemas,
90-
SkipResources: req.SkipResources,
84+
OrganizationID: organization.ID,
9185
})
9286
if err != nil {
9387
httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{

‎coderd/provisionerjobs_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package coderd_test
22

33
import (
44
"context"
5+
"fmt"
56
"net/http"
67
"testing"
78
"time"
@@ -70,6 +71,7 @@ func TestPostProvisionerImportJobByOrganization(t *testing.T) {
7071
},
7172
},
7273
}},
74+
Provision: echo.ProvisionComplete,
7375
})
7476
require.NoError(t, err)
7577
file, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, data)
@@ -84,10 +86,10 @@ func TestPostProvisionerImportJobByOrganization(t *testing.T) {
8486
SourceScheme: database.ParameterSourceSchemeData,
8587
DestinationScheme: database.ParameterDestinationSchemeProvisionerVariable,
8688
}},
87-
SkipResources: true,
8889
})
8990
require.NoError(t, err)
90-
coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
91+
job = coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
92+
fmt.Printf("Job %+v\n", job)
9193
values, err := client.ProvisionerJobParameterValues(context.Background(), user.Organization, job.ID)
9294
require.NoError(t, err)
9395
require.Equal(t, "somevalue", values[0].SourceValue)

0 commit comments

Comments
 (0)