Skip to content

Commit 6e7f232

Browse files
EmyrkEmyrk
committed
builtin provisioner supporting multiple types
1 parent 342aa84 commit 6e7f232

File tree

4 files changed

+132
-106
lines changed

4 files changed

+132
-106
lines changed

‎cli/server.go

Lines changed: 78 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -945,26 +945,21 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
945945
defer provisionerdWaitGroup.Wait()
946946
provisionerdMetrics := provisionerd.NewMetrics(options.PrometheusRegistry)
947947

948-
// Create a list of daemon types. The length is the total number of built-in provisioners, and
949-
// the slice value is the type for each. This is just an easy way to pass the types
950-
// to a single loop. Ensuring each provisioner has a unique suffix with their index.
951-
daemons := make([]codersdk.ProvisionerType, 0)
952-
for i := int64(0); i < vals.Provisioner.DaemonsTerraform.Value(); i++ {
953-
daemons = append(daemons, codersdk.ProvisionerTypeTerraform)
948+
// Built in provisioner daemons will support the same types.
949+
// By default, this is the slice {"terraform"}
950+
provisionerTypes := make([]codersdk.ProvisionerType, 0)
951+
for _, pt := range vals.Provisioner.DaemonTypes {
952+
provisionerTypes = append(provisionerTypes, codersdk.ProvisionerType(pt))
954953
}
955-
for i := int64(0); i < vals.Provisioner.DaemonsEcho.Value(); i++ {
956-
daemons = append(daemons, codersdk.ProvisionerTypeEcho)
957-
}
958-
959-
for i, provisionerType := range daemons {
954+
for i := int64(0); i < vals.Provisioner.Daemons.Value(); i++ {
960955
suffix := fmt.Sprintf("%d", i)
961956
// The suffix is added to the hostname, so we may need to trim to fit into
962957
// the 64 character limit.
963958
hostname := stringutil.Truncate(cliutil.Hostname(), 63-len(suffix))
964959
name := fmt.Sprintf("%s-%s", hostname, suffix)
965960
daemonCacheDir := filepath.Join(cacheDir, fmt.Sprintf("provisioner-%d", i))
966961
daemon, err := newProvisionerDaemon(
967-
ctx, coderAPI, provisionerdMetrics, logger, vals, daemonCacheDir, errCh, &provisionerdWaitGroup, name, provisionerType,
962+
ctx, coderAPI, provisionerdMetrics, logger, vals, daemonCacheDir, errCh, &provisionerdWaitGroup, name, provisionerTypes,
968963
)
969964
if err != nil {
970965
return xerrors.Errorf("create provisioner daemon: %w", err)
@@ -1352,7 +1347,7 @@ func newProvisionerDaemon(
13521347
errCh chan error,
13531348
wg *sync.WaitGroup,
13541349
name string,
1355-
provisionerType codersdk.ProvisionerType,
1350+
provisionerTypes []codersdk.ProvisionerType,
13561351
) (srv *provisionerd.Server, err error) {
13571352
ctx, cancel := context.WithCancel(ctx)
13581353
defer func() {
@@ -1372,82 +1367,88 @@ func newProvisionerDaemon(
13721367
return nil, xerrors.Errorf("mkdir work dir: %w", err)
13731368
}
13741369

1370+
// Omit any duplicates
1371+
provisionerTypes = slice.Unique(provisionerTypes)
1372+
1373+
// Populate the connector with the supported types.
13751374
connector := provisionerd.LocalProvisioners{}
1376-
switch provisionerType {
1377-
case codersdk.ProvisionerTypeEcho:
1378-
echoClient, echoServer := drpc.MemTransportPipe()
1379-
wg.Add(1)
1380-
go func() {
1381-
defer wg.Done()
1382-
<-ctx.Done()
1383-
_ = echoClient.Close()
1384-
_ = echoServer.Close()
1385-
}()
1386-
wg.Add(1)
1387-
go func() {
1388-
defer wg.Done()
1389-
defer cancel()
1375+
for _, provisionerType := range provisionerTypes {
1376+
switch provisionerType {
1377+
case codersdk.ProvisionerTypeEcho:
1378+
echoClient, echoServer := drpc.MemTransportPipe()
1379+
wg.Add(1)
1380+
go func() {
1381+
defer wg.Done()
1382+
<-ctx.Done()
1383+
_ = echoClient.Close()
1384+
_ = echoServer.Close()
1385+
}()
1386+
wg.Add(1)
1387+
go func() {
1388+
defer wg.Done()
1389+
defer cancel()
13901390

1391-
err := echo.Serve(ctx, &provisionersdk.ServeOptions{
1392-
Listener: echoServer,
1393-
WorkDirectory: workDir,
1394-
Logger: logger.Named("echo"),
1395-
})
1396-
if err != nil {
1397-
select {
1398-
case errCh <- err:
1399-
default:
1391+
err := echo.Serve(ctx, &provisionersdk.ServeOptions{
1392+
Listener: echoServer,
1393+
WorkDirectory: workDir,
1394+
Logger: logger.Named("echo"),
1395+
})
1396+
if err != nil {
1397+
select {
1398+
case errCh <- err:
1399+
default:
1400+
}
14001401
}
1402+
}()
1403+
connector[string(database.ProvisionerTypeEcho)] = sdkproto.NewDRPCProvisionerClient(echoClient)
1404+
case codersdk.ProvisionerTypeTerraform:
1405+
tfDir := filepath.Join(cacheDir, "tf")
1406+
err = os.MkdirAll(tfDir, 0o700)
1407+
if err != nil {
1408+
return nil, xerrors.Errorf("mkdir terraform dir: %w", err)
14011409
}
1402-
}()
1403-
connector[string(database.ProvisionerTypeEcho)] = sdkproto.NewDRPCProvisionerClient(echoClient)
1404-
case codersdk.ProvisionerTypeTerraform:
1405-
tfDir := filepath.Join(cacheDir, "tf")
1406-
err = os.MkdirAll(tfDir, 0o700)
1407-
if err != nil {
1408-
return nil, xerrors.Errorf("mkdir terraform dir: %w", err)
1409-
}
14101410

1411-
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
1412-
terraformClient, terraformServer := drpc.MemTransportPipe()
1413-
wg.Add(1)
1414-
go func() {
1415-
defer wg.Done()
1416-
<-ctx.Done()
1417-
_ = terraformClient.Close()
1418-
_ = terraformServer.Close()
1419-
}()
1420-
wg.Add(1)
1421-
go func() {
1422-
defer wg.Done()
1423-
defer cancel()
1424-
1425-
err := terraform.Serve(ctx, &terraform.ServeOptions{
1426-
ServeOptions: &provisionersdk.ServeOptions{
1427-
Listener: terraformServer,
1428-
Logger: logger.Named("terraform"),
1429-
WorkDirectory: workDir,
1430-
},
1431-
CachePath: tfDir,
1432-
Tracer: tracer,
1433-
})
1434-
if err != nil && !xerrors.Is(err, context.Canceled) {
1435-
select {
1436-
case errCh <- err:
1437-
default:
1411+
tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
1412+
terraformClient, terraformServer := drpc.MemTransportPipe()
1413+
wg.Add(1)
1414+
go func() {
1415+
defer wg.Done()
1416+
<-ctx.Done()
1417+
_ = terraformClient.Close()
1418+
_ = terraformServer.Close()
1419+
}()
1420+
wg.Add(1)
1421+
go func() {
1422+
defer wg.Done()
1423+
defer cancel()
1424+
1425+
err := terraform.Serve(ctx, &terraform.ServeOptions{
1426+
ServeOptions: &provisionersdk.ServeOptions{
1427+
Listener: terraformServer,
1428+
Logger: logger.Named("terraform"),
1429+
WorkDirectory: workDir,
1430+
},
1431+
CachePath: tfDir,
1432+
Tracer: tracer,
1433+
})
1434+
if err != nil && !xerrors.Is(err, context.Canceled) {
1435+
select {
1436+
case errCh <- err:
1437+
default:
1438+
}
14381439
}
1439-
}
1440-
}()
1440+
}()
14411441

1442-
connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
1443-
default:
1444-
return nil, fmt.Errorf("unknown provisioner type %q", provisionerType)
1442+
connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
1443+
default:
1444+
return nil, fmt.Errorf("unknown provisioner type %q", provisionerType)
1445+
}
14451446
}
14461447

14471448
return provisionerd.New(func(dialCtx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
14481449
// This debounces calls to listen every second. Read the comment
14491450
// in provisionerdserver.go to learn more!
1450-
return coderAPI.CreateInMemoryProvisionerDaemon(dialCtx, name, []codersdk.ProvisionerType{provisionerType})
1451+
return coderAPI.CreateInMemoryProvisionerDaemon(dialCtx, name, provisionerTypes)
14511452
}, &provisionerd.Options{
14521453
Logger: logger.Named(fmt.Sprintf("provisionerd-%s", name)),
14531454
UpdateInterval: time.Second,

‎cli/server_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1367,8 +1367,8 @@ func TestServer(t *testing.T) {
13671367
"--in-memory",
13681368
"--http-address", ":0",
13691369
"--access-url", "http://example.com",
1370-
"--provisioner-daemons=0",
1371-
"--provisioner-daemons-echo=3",
1370+
"--provisioner-daemons=3",
1371+
"--provisioner-types=echo",
13721372
"--log-human", fiName,
13731373
)
13741374
clitest.Start(t, root)
@@ -1386,8 +1386,8 @@ func TestServer(t *testing.T) {
13861386
"--in-memory",
13871387
"--http-address", ":0",
13881388
"--access-url", "http://example.com",
1389-
"--provisioner-daemons=0",
1390-
"--provisioner-daemons-echo=3",
1389+
"--provisioner-daemons=3",
1390+
"--provisioner-types=echo",
13911391
"--log-human", fi,
13921392
)
13931393
clitest.Start(t, root)
@@ -1405,8 +1405,8 @@ func TestServer(t *testing.T) {
14051405
"--in-memory",
14061406
"--http-address", ":0",
14071407
"--access-url", "http://example.com",
1408-
"--provisioner-daemons=0",
1409-
"--provisioner-daemons-echo=3",
1408+
"--provisioner-daemons=3",
1409+
"--provisioner-types=echo",
14101410
"--log-json", fi,
14111411
)
14121412
clitest.Start(t, root)
@@ -1427,8 +1427,8 @@ func TestServer(t *testing.T) {
14271427
"--in-memory",
14281428
"--http-address", ":0",
14291429
"--access-url", "http://example.com",
1430-
"--provisioner-daemons-echo=3",
1431-
"--provisioner-daemons=0",
1430+
"--provisioner-daemons=3",
1431+
"--provisioner-types=echo",
14321432
"--log-stackdriver", fi,
14331433
)
14341434
// Attach pty so we get debug output from the command if this test
@@ -1463,8 +1463,8 @@ func TestServer(t *testing.T) {
14631463
"--in-memory",
14641464
"--http-address", ":0",
14651465
"--access-url", "http://example.com",
1466-
"--provisioner-daemons-echo=3",
1467-
"--provisioner-daemons=0",
1466+
"--provisioner-daemons=3",
1467+
"--provisioner-types=echo",
14681468
"--log-human", fi1,
14691469
"--log-json", fi2,
14701470
"--log-stackdriver", fi3,

‎codersdk/deployment.go

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -406,14 +406,13 @@ type ExternalAuthConfig struct {
406406
}
407407

408408
type ProvisionerConfig struct {
409-
// DaemonsTerraform is the number of built-in terraform provisioners. The
410-
// json is 'daemons' for legacy reasons.
411-
DaemonsTerraform serpent.Int64 `json:"daemons" typescript:",notnull"`
412-
DaemonsEcho serpent.Int64 `json:"daemons_echo" typescript:",notnull"`
413-
DaemonPollInterval serpent.Duration `json:"daemon_poll_interval" typescript:",notnull"`
414-
DaemonPollJitter serpent.Duration `json:"daemon_poll_jitter" typescript:",notnull"`
415-
ForceCancelInterval serpent.Duration `json:"force_cancel_interval" typescript:",notnull"`
416-
DaemonPSK serpent.String `json:"daemon_psk" typescript:",notnull"`
409+
// Daemons is the number of built-in terraform provisioners.
410+
Daemons serpent.Int64 `json:"daemons" typescript:",notnull"`
411+
DaemonTypes serpent.StringArray `json:"daemons_echo" typescript:",notnull"`
412+
DaemonPollInterval serpent.Duration `json:"daemon_poll_interval" typescript:",notnull"`
413+
DaemonPollJitter serpent.Duration `json:"daemon_poll_jitter" typescript:",notnull"`
414+
ForceCancelInterval serpent.Duration `json:"force_cancel_interval" typescript:",notnull"`
415+
DaemonPSK serpent.String `json:"daemon_psk" typescript:",notnull"`
417416
}
418417

419418
type RateLimitConfig struct {
@@ -1406,24 +1405,39 @@ when required by your organization's security policy.`,
14061405
// Provisioner settings
14071406
{
14081407
Name: "Provisioner Daemons",
1409-
Description: "Number of terraform provisioner daemons to create on start. If builds are stuck in queued state for a long time, consider increasing this.",
1408+
Description: "Number of provisioner daemons to create on start. If builds are stuck in queued state for a long time, consider increasing this.",
14101409
Flag: "provisioner-daemons",
14111410
Env: "CODER_PROVISIONER_DAEMONS",
14121411
Default: "3",
1413-
Value: &c.Provisioner.DaemonsTerraform,
1412+
Value: &c.Provisioner.Daemons,
14141413
Group: &deploymentGroupProvisioning,
14151414
YAML: "daemons",
14161415
},
14171416
{
1418-
Name: "Echo Provisioner",
1419-
Description: "Number of built-in echo provisioners to create on start. Can be done alongside actual terraform provisioners. This is for E2E tests.",
1420-
Flag: "provisioner-daemons-echo",
1421-
Env: "CODER_PROVISIONER_DAEMONS_ECHO",
1422-
Hidden: true,
1423-
Default: "0",
1424-
Value: &c.Provisioner.DaemonsEcho,
1425-
Group: &deploymentGroupProvisioning,
1426-
YAML: "daemonsEcho",
1417+
Name: "Provisioner Daemon Types",
1418+
Description: fmt.Sprintf("The supported job types for the built-in provisioners. By default, this is only the terraform type. Supported types: %s.",
1419+
strings.Join([]string{
1420+
string(ProvisionerTypeTerraform), string(ProvisionerTypeEcho),
1421+
}, ",")),
1422+
Flag: "provisioner-types",
1423+
Env: "CODER_PROVISIONER_TYPES",
1424+
Hidden: true,
1425+
Default: string(ProvisionerTypeTerraform),
1426+
Value: serpent.Validate(&c.Provisioner.DaemonTypes, func(values *serpent.StringArray) error {
1427+
if values == nil {
1428+
return nil
1429+
}
1430+
1431+
for _, value := range *values {
1432+
if err := ProvisionerTypeValid(value); err != nil {
1433+
return err
1434+
}
1435+
}
1436+
1437+
return nil
1438+
}),
1439+
Group: &deploymentGroupProvisioning,
1440+
YAML: "daemonsEcho",
14271441
},
14281442
{
14291443
Name: "Poll Interval",

‎codersdk/organizations.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,17 @@ const (
2727
ProvisionerTypeTerraform ProvisionerType = "terraform"
2828
)
2929

30+
// ProvisionerTypeValid accepts string or ProvisionerType for easier usage.
31+
// Will validate the enum is in the set.
32+
func ProvisionerTypeValid[T ProvisionerType | string](pt T) error {
33+
switch string(pt) {
34+
case string(ProvisionerTypeEcho), string(ProvisionerTypeTerraform):
35+
return nil
36+
default:
37+
return fmt.Errorf("provisioner type '%s' is not supported", pt)
38+
}
39+
}
40+
3041
// Organization is the JSON representation of a Coder organization.
3142
type Organization struct {
3243
ID uuid.UUID `table:"id" json:"id" validate:"required" format:"uuid"`

0 commit comments

Comments
 (0)