Skip to content

Commit 94a3e3a

Browse files
EmyrkEmyrk
authored
chore: allow terraform & echo built-in provisioners (#13121)
* chore: allow terraform & echo built-in provisioners Built-in provisioners serve all specified types. This allows running terraform, echo, or both in built in. The cli flag to control the types is hidden by default, to be used primarily for testing purposes.
1 parent 7873c96 commit 94a3e3a

File tree

13 files changed

+177
-115
lines changed

13 files changed

+177
-115
lines changed

‎cli/server.go

Lines changed: 79 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -944,6 +944,13 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
944944
var provisionerdWaitGroup sync.WaitGroup
945945
defer provisionerdWaitGroup.Wait()
946946
provisionerdMetrics := provisionerd.NewMetrics(options.PrometheusRegistry)
947+
948+
// Built in provisioner daemons will support the same types.
949+
// By default, this is the slice {"terraform"}
950+
provisionerTypes := make([]codersdk.ProvisionerType, 0)
951+
for _, pt := range vals.Provisioner.DaemonTypes {
952+
provisionerTypes = append(provisionerTypes, codersdk.ProvisionerType(pt))
953+
}
947954
for i := int64(0); i < vals.Provisioner.Daemons.Value(); i++ {
948955
suffix := fmt.Sprintf("%d", i)
949956
// The suffix is added to the hostname, so we may need to trim to fit into
@@ -952,7 +959,7 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
952959
name := fmt.Sprintf("%s-%s", hostname, suffix)
953960
daemonCacheDir := filepath.Join(cacheDir, fmt.Sprintf("provisioner-%d", i))
954961
daemon, err := newProvisionerDaemon(
955-
ctx, coderAPI, provisionerdMetrics, logger, vals, daemonCacheDir, errCh, &provisionerdWaitGroup, name,
962+
ctx, coderAPI, provisionerdMetrics, logger, vals, daemonCacheDir, errCh, &provisionerdWaitGroup, name, provisionerTypes,
956963
)
957964
if err != nil {
958965
return xerrors.Errorf("create provisioner daemon: %w", err)
@@ -1340,6 +1347,7 @@ func newProvisionerDaemon(
13401347
errCh chan error,
13411348
wg *sync.WaitGroup,
13421349
name string,
1350+
provisionerTypes []codersdk.ProvisionerType,
13431351
) (srv *provisionerd.Server, err error) {
13441352
ctx, cancel := context.WithCancel(ctx)
13451353
defer func() {
@@ -1359,79 +1367,88 @@ func newProvisionerDaemon(
13591367
return nil, xerrors.Errorf("mkdir work dir: %w", err)
13601368
}
13611369

1370+
// Omit any duplicates
1371+
provisionerTypes = slice.Unique(provisionerTypes)
1372+
1373+
// Populate the connector with the supported types.
13621374
connector := provisionerd.LocalProvisioners{}
1363-
if cfg.Provisioner.DaemonsEcho {
1364-
echoClient, echoServer := drpc.MemTransportPipe()
1365-
wg.Add(1)
1366-
go func() {
1367-
defer wg.Done()
1368-
<-ctx.Done()
1369-
_ = echoClient.Close()
1370-
_ = echoServer.Close()
1371-
}()
1372-
wg.Add(1)
1373-
go func() {
1374-
defer wg.Done()
1375-
defer cancel()
1375+
for _, provisionerType := range provisionerTypes {
1376+
switch provisionerType {
1377+
case codersdk.ProvisionerTypeEcho:
1378+
echoClient, echoServer := drpc.MemTransportPipe()
1379+
wg.Add(1)
1380+
go func() {
1381+
defer wg.Done()
1382+
<-ctx.Done()
1383+
_ = echoClient.Close()
1384+
_ = echoServer.Close()
1385+
}()
1386+
wg.Add(1)
1387+
go func() {
1388+
defer wg.Done()
1389+
defer cancel()
13761390

1377-
err := echo.Serve(ctx, &provisionersdk.ServeOptions{
1378-
Listener: echoServer,
1379-
WorkDirectory: workDir,
1380-
Logger: logger.Named("echo"),
1381-
})
1382-
if err != nil {
1383-
select {
1384-
case errCh <- err:
1385-
default:
1391+
err := echo.Serve(ctx, &provisionersdk.ServeOptions{
1392+
Listener: echoServer,
1393+
WorkDirectory: workDir,
1394+
Logger: logger.Named("echo"),
1395+
})
1396+
if err != nil {
1397+
select {
1398+
case errCh <- err:
1399+
default:
1400+
}
13861401
}
1402+
}()
1403+
connector[string(database.ProvisionerTypeEcho)] = sdkproto.NewDRPCProvisionerClient(echoClient)
1404+
case codersdk.ProvisionerTypeTerraform:
1405+
tfDir := filepath.Join(cacheDir, "tf")
1406+
err = os.MkdirAll(tfDir, 0o700)
1407+
if err != nil {
1408+
return nil, xerrors.Errorf("mkdir terraform dir: %w", err)
13871409
}
1388-
}()
1389-
connector[string(database.ProvisionerTypeEcho)] = sdkproto.NewDRPCProvisionerClient(echoClient)
1390-
} else {
1391-
tfDir := filepath.Join(cacheDir, "tf")
1392-
err = os.MkdirAll(tfDir, 0o700)
1393-
if err != nil {
1394-
return nil, xerrors.Errorf("mkdir terraform dir: %w", err)
1395-
}
13961410

1397-
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
1398-
terraformClient, terraformServer := drpc.MemTransportPipe()
1399-
wg.Add(1)
1400-
go func() {
1401-
defer wg.Done()
1402-
<-ctx.Done()
1403-
_ = terraformClient.Close()
1404-
_ = terraformServer.Close()
1405-
}()
1406-
wg.Add(1)
1407-
go func() {
1408-
defer wg.Done()
1409-
defer cancel()
1410-
1411-
err := terraform.Serve(ctx, &terraform.ServeOptions{
1412-
ServeOptions: &provisionersdk.ServeOptions{
1413-
Listener: terraformServer,
1414-
Logger: logger.Named("terraform"),
1415-
WorkDirectory: workDir,
1416-
},
1417-
CachePath: tfDir,
1418-
Tracer: tracer,
1419-
})
1420-
if err != nil && !xerrors.Is(err, context.Canceled) {
1421-
select {
1422-
case errCh <- err:
1423-
default:
1411+
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
1412+
terraformClient, terraformServer := drpc.MemTransportPipe()
1413+
wg.Add(1)
1414+
go func() {
1415+
defer wg.Done()
1416+
<-ctx.Done()
1417+
_ = terraformClient.Close()
1418+
_ = terraformServer.Close()
1419+
}()
1420+
wg.Add(1)
1421+
go func() {
1422+
defer wg.Done()
1423+
defer cancel()
1424+
1425+
err := terraform.Serve(ctx, &terraform.ServeOptions{
1426+
ServeOptions: &provisionersdk.ServeOptions{
1427+
Listener: terraformServer,
1428+
Logger: logger.Named("terraform"),
1429+
WorkDirectory: workDir,
1430+
},
1431+
CachePath: tfDir,
1432+
Tracer: tracer,
1433+
})
1434+
if err != nil && !xerrors.Is(err, context.Canceled) {
1435+
select {
1436+
case errCh <- err:
1437+
default:
1438+
}
14241439
}
1425-
}
1426-
}()
1440+
}()
14271441

1428-
connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
1442+
connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
1443+
default:
1444+
return nil, fmt.Errorf("unknown provisioner type %q", provisionerType)
1445+
}
14291446
}
14301447

14311448
return provisionerd.New(func(dialCtx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
14321449
// This debounces calls to listen every second. Read the comment
14331450
// in provisionerdserver.go to learn more!
1434-
return coderAPI.CreateInMemoryProvisionerDaemon(dialCtx, name)
1451+
return coderAPI.CreateInMemoryProvisionerDaemon(dialCtx, name, provisionerTypes)
14351452
}, &provisionerd.Options{
14361453
Logger: logger.Named(fmt.Sprintf("provisionerd-%s", name)),
14371454
UpdateInterval: time.Second,

‎cli/server_test.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,7 +1367,8 @@ func TestServer(t *testing.T) {
13671367
"--in-memory",
13681368
"--http-address", ":0",
13691369
"--access-url", "http://example.com",
1370-
"--provisioner-daemons-echo",
1370+
"--provisioner-daemons=3",
1371+
"--provisioner-types=echo",
13711372
"--log-human", fiName,
13721373
)
13731374
clitest.Start(t, root)
@@ -1385,7 +1386,8 @@ func TestServer(t *testing.T) {
13851386
"--in-memory",
13861387
"--http-address", ":0",
13871388
"--access-url", "http://example.com",
1388-
"--provisioner-daemons-echo",
1389+
"--provisioner-daemons=3",
1390+
"--provisioner-types=echo",
13891391
"--log-human", fi,
13901392
)
13911393
clitest.Start(t, root)
@@ -1403,7 +1405,8 @@ func TestServer(t *testing.T) {
14031405
"--in-memory",
14041406
"--http-address", ":0",
14051407
"--access-url", "http://example.com",
1406-
"--provisioner-daemons-echo",
1408+
"--provisioner-daemons=3",
1409+
"--provisioner-types=echo",
14071410
"--log-json", fi,
14081411
)
14091412
clitest.Start(t, root)
@@ -1424,7 +1427,8 @@ func TestServer(t *testing.T) {
14241427
"--in-memory",
14251428
"--http-address", ":0",
14261429
"--access-url", "http://example.com",
1427-
"--provisioner-daemons-echo",
1430+
"--provisioner-daemons=3",
1431+
"--provisioner-types=echo",
14281432
"--log-stackdriver", fi,
14291433
)
14301434
// Attach pty so we get debug output from the command if this test
@@ -1459,7 +1463,8 @@ func TestServer(t *testing.T) {
14591463
"--in-memory",
14601464
"--http-address", ":0",
14611465
"--access-url", "http://example.com",
1462-
"--provisioner-daemons-echo",
1466+
"--provisioner-daemons=3",
1467+
"--provisioner-types=echo",
14631468
"--log-human", fi1,
14641469
"--log-json", fi2,
14651470
"--log-stackdriver", fi3,

‎cli/testdata/server-config.yaml.golden

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -379,10 +379,11 @@ provisioning:
379379
# state for a long time, consider increasing this.
380380
# (default: 3, type: int)
381381
daemons: 3
382-
# Whether to use echo provisioner daemons instead of Terraform. This is for E2E
383-
# tests.
384-
# (default: false, type: bool)
385-
daemonsEcho: false
382+
# The supported job types for the built-in provisioners. By default, this is only
383+
# the terraform type. Supported types: terraform,echo.
384+
# (default: terraform, type: string-array)
385+
daemonTypes:
386+
- terraform
386387
# Deprecated and ignored.
387388
# (default: 1s, type: duration)
388389
daemonPollInterval: 1s

‎coderd/apidoc/docs.go

Lines changed: 7 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json

Lines changed: 7 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/coderd.go

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1348,7 +1348,7 @@ func compressHandler(h http.Handler) http.Handler {
13481348

13491349
// CreateInMemoryProvisionerDaemon is an in-memory connection to a provisionerd.
13501350
// Useful when starting coderd and provisionerd in the same process.
1351-
func (api *API) CreateInMemoryProvisionerDaemon(dialCtx context.Context, name string) (client proto.DRPCProvisionerDaemonClient, err error) {
1351+
func (api *API) CreateInMemoryProvisionerDaemon(dialCtx context.Context, name string, provisionerTypes []codersdk.ProvisionerType) (client proto.DRPCProvisionerDaemonClient, err error) {
13521352
tracer := api.TracerProvider.Tracer(tracing.TracerName)
13531353
clientSession, serverSession := drpc.MemTransportPipe()
13541354
defer func() {
@@ -1365,18 +1365,21 @@ func (api *API) CreateInMemoryProvisionerDaemon(dialCtx context.Context, name st
13651365
return nil, xerrors.Errorf("unable to fetch default org for in memory provisioner: %w", err)
13661366
}
13671367

1368+
dbTypes := make([]database.ProvisionerType, 0, len(provisionerTypes))
1369+
for _, tp := range provisionerTypes {
1370+
dbTypes = append(dbTypes, database.ProvisionerType(tp))
1371+
}
1372+
13681373
//nolint:gocritic // in-memory provisioners are owned by system
13691374
daemon, err := api.Database.UpsertProvisionerDaemon(dbauthz.AsSystemRestricted(dialCtx), database.UpsertProvisionerDaemonParams{
13701375
Name: name,
13711376
OrganizationID: defaultOrg.ID,
13721377
CreatedAt: dbtime.Now(),
1373-
Provisioners: []database.ProvisionerType{
1374-
database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform,
1375-
},
1376-
Tags: provisionersdk.MutateTags(uuid.Nil, nil),
1377-
LastSeenAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
1378-
Version: buildinfo.Version(),
1379-
APIVersion: proto.CurrentVersion.String(),
1378+
Provisioners: dbTypes,
1379+
Tags: provisionersdk.MutateTags(uuid.Nil, nil),
1380+
LastSeenAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
1381+
Version: buildinfo.Version(),
1382+
APIVersion: proto.CurrentVersion.String(),
13801383
})
13811384
if err != nil {
13821385
return nil, xerrors.Errorf("failed to create in-memory provisioner daemon: %w", err)

‎coderd/coderdtest/coderdtest.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -578,7 +578,7 @@ func NewProvisionerDaemon(t testing.TB, coderAPI *coderd.API) io.Closer {
578578
}()
579579

580580
daemon := provisionerd.New(func(dialCtx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) {
581-
return coderAPI.CreateInMemoryProvisionerDaemon(dialCtx, "test")
581+
return coderAPI.CreateInMemoryProvisionerDaemon(dialCtx, "test", []codersdk.ProvisionerType{codersdk.ProvisionerTypeEcho})
582582
}, &provisionerd.Options{
583583
Logger: coderAPI.Logger.Named("provisionerd").Leveled(slog.LevelDebug),
584584
UpdateInterval: 250 * time.Millisecond,

‎codersdk/deployment.go

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -406,12 +406,13 @@ type ExternalAuthConfig struct {
406406
}
407407

408408
type ProvisionerConfig struct {
409-
Daemons serpent.Int64 `json:"daemons" typescript:",notnull"`
410-
DaemonsEcho serpent.Bool `json:"daemons_echo" typescript:",notnull"`
411-
DaemonPollInterval serpent.Duration `json:"daemon_poll_interval" typescript:",notnull"`
412-
DaemonPollJitter serpent.Duration `json:"daemon_poll_jitter" typescript:",notnull"`
413-
ForceCancelInterval serpent.Duration `json:"force_cancel_interval" typescript:",notnull"`
414-
DaemonPSK serpent.String `json:"daemon_psk" typescript:",notnull"`
409+
// Daemons is the number of built-in terraform provisioners.
410+
Daemons serpent.Int64 `json:"daemons" typescript:",notnull"`
411+
DaemonTypes serpent.StringArray `json:"daemon_types" typescript:",notnull"`
412+
DaemonPollInterval serpent.Duration `json:"daemon_poll_interval" typescript:",notnull"`
413+
DaemonPollJitter serpent.Duration `json:"daemon_poll_jitter" typescript:",notnull"`
414+
ForceCancelInterval serpent.Duration `json:"force_cancel_interval" typescript:",notnull"`
415+
DaemonPSK serpent.String `json:"daemon_psk" typescript:",notnull"`
415416
}
416417

417418
type RateLimitConfig struct {
@@ -1413,15 +1414,30 @@ when required by your organization's security policy.`,
14131414
YAML: "daemons",
14141415
},
14151416
{
1416-
Name: "Echo Provisioner",
1417-
Description: "Whether to use echo provisioner daemons instead of Terraform. This is for E2E tests.",
1418-
Flag: "provisioner-daemons-echo",
1419-
Env: "CODER_PROVISIONER_DAEMONS_ECHO",
1420-
Hidden: true,
1421-
Default: "false",
1422-
Value: &c.Provisioner.DaemonsEcho,
1423-
Group: &deploymentGroupProvisioning,
1424-
YAML: "daemonsEcho",
1417+
Name: "Provisioner Daemon Types",
1418+
Description: fmt.Sprintf("The supported job types for the built-in provisioners. By default, this is only the terraform type. Supported types: %s.",
1419+
strings.Join([]string{
1420+
string(ProvisionerTypeTerraform), string(ProvisionerTypeEcho),
1421+
}, ",")),
1422+
Flag: "provisioner-types",
1423+
Env: "CODER_PROVISIONER_TYPES",
1424+
Hidden: true,
1425+
Default: string(ProvisionerTypeTerraform),
1426+
Value: serpent.Validate(&c.Provisioner.DaemonTypes, func(values *serpent.StringArray) error {
1427+
if values == nil {
1428+
return nil
1429+
}
1430+
1431+
for _, value := range *values {
1432+
if err := ProvisionerTypeValid(value); err != nil {
1433+
return err
1434+
}
1435+
}
1436+
1437+
return nil
1438+
}),
1439+
Group: &deploymentGroupProvisioning,
1440+
YAML: "daemonTypes",
14251441
},
14261442
{
14271443
Name: "Poll Interval",

0 commit comments

Comments
 (0)