From a65b90b09ed67699d60c91127ace47f7aa49eb95 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 28 Feb 2024 13:19:52 +0000 Subject: [PATCH 1/4] chore(coderd): add more test cases to TestAcquirer_MatchTags, generate table for docs --- coderd/provisionerdserver/acquirer_test.go | 151 +++++++++++++++++---- 1 file changed, 121 insertions(+), 30 deletions(-) diff --git a/coderd/provisionerdserver/acquirer_test.go b/coderd/provisionerdserver/acquirer_test.go index 088fb68a5acaf..f6182ae0b4e8e 100644 --- a/coderd/provisionerdserver/acquirer_test.go +++ b/coderd/provisionerdserver/acquirer_test.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "encoding/json" + "fmt" + "strings" "sync" "testing" "time" @@ -54,7 +56,7 @@ func TestAcquirer_Single(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -82,7 +84,7 @@ func TestAcquirer_MultipleSameDomain(t *testing.T) { workerIDs := make(map[uuid.UUID]bool) pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } for i := 0; i < 10; i++ { wID := uuid.New() @@ -125,7 +127,7 @@ func TestAcquirer_WaitsOnNoJobs(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -147,10 +149,10 @@ func TestAcquirer_WaitsOnNoJobs(t *testing.T) { "strong": "bad", }) postJob(t, ps, database.ProvisionerTypeEcho, provisionerdserver.Tags{ - "foo": "fighters", + "environment": "fighters", }) postJob(t, ps, database.ProvisionerTypeTerraform, provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", }) acquiree.requireBlocked() @@ -176,7 +178,7 @@ func TestAcquirer_RetriesPending(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -268,7 +270,7 @@ func TestAcquirer_BackupPoll(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -294,7 +296,7 @@ func TestAcquirer_UnblockOnCancel(t *testing.T) { pt := []database.ProvisionerType{database.ProvisionerTypeEcho} worker0 := uuid.New() tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree0 := newTestAcquiree(t, worker0, pt, tags) worker1 := uuid.New() @@ -324,10 +326,7 @@ func TestAcquirer_MatchTags(t *testing.T) { t.Skip("skipping this test due to -short") } - someID := uuid.NewString() - someOtherID := uuid.NewString() - - for _, tt := range []struct { + testCases := []struct { name string provisionerJobTags map[string]string acquireJobTags map[string]string @@ -341,69 +340,118 @@ func TestAcquirer_MatchTags(t *testing.T) { }, { name: "untagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: false, }, { name: "tagged provisioner and untagged job", provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: false, }, { name: "tagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: true, }, { name: "tagged provisioner and double-tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: false, }, { name: "double-tagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, expectAcquire: true, }, { name: "double-tagged provisioner and double-tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, expectAcquire: true, }, { name: "owner-scoped provisioner and untagged job", provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, - acquireJobTags: map[string]string{"scope": "owner", "owner": someID}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, expectAcquire: false, }, { name: "owner-scoped provisioner and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": someID}, - acquireJobTags: map[string]string{"scope": "owner", "owner": someID}, + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, expectAcquire: true, }, + { name: "owner-scoped provisioner and different owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": someOtherID}, - acquireJobTags: map[string]string{"scope": "owner", "owner": someID}, + provisionerJobTags: map[string]string{"scope": "owner", "owner": "bbb"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, expectAcquire: false, }, { name: "org-scoped provisioner and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": someID}, + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, + expectAcquire: false, + }, + { + name: "owner-scoped provisioner and org-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: false, }, - } { + { + name: "owner-scoped provisioner with tags and owner-scoped job", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: true, + }, + { + name: "owner-scoped provisioner and owner-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + expectAcquire: false, + }, + { + name: "owner-scoped provisioner with tags and owner-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: true, + }, + { + name: "owner-scoped provisioner with multiple tags and owner-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, + { + name: "owner-scoped provisioner with tags and owner-scoped job with multiple tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: false, + }, + { + name: "owner-scoped provisioner with tags and owner-scoped job with multiple tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, + { + name: "owner-scoped provisioner with tags and owner-scoped job with differing tags", + provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "new_york"}, + acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: false, + }, + } + for _, tt := range testCases { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitShort/2) // NOTE: explicitly not using fake store for this test. db, ps := dbtestutil.NewDB(t) @@ -443,6 +491,43 @@ func TestAcquirer_MatchTags(t *testing.T) { } }) } + + t.Run("GenTable", func(t *testing.T) { + t.Parallel() + // Generate a table that can be copy-pasted into docs/admin/provisioners.md + lines := []string{ + "\n", + "| Provisioner Tags | Job Tags | Can Run Job? |", + "|------------------|----------|--------------|", + } + // turn the JSON map into k=v for readability + kvs := func(m map[string]string) string { + ss := make([]string, 0, len(m)) + // always start with scope and owner + ss = append(ss, "scope"+"="+m["scope"]) + ss = append(ss, "owner"+"="+m["owner"]) + for k, v := range m { + if k == "scope" || k == "owner" { + continue + } + if len(v) == 0 { + v = `""` + } + ss = append(ss, k+"="+v) + } + return strings.Join(ss, " ") + } + for _, tt := range testCases { + acquire := "✅" + if !tt.expectAcquire { + acquire = "❌" + } + s := fmt.Sprintf("| %s | %s | %s |", kvs(tt.acquireJobTags), kvs(tt.provisionerJobTags), acquire) + lines = append(lines, s) + } + t.Logf("You can paste this into docs/admin/provisioners.md") + t.Logf(strings.Join(lines, "\n")) + }) } func postJob(t *testing.T, ps pubsub.Pubsub, pt database.ProvisionerType, tags provisionerdserver.Tags) { @@ -641,3 +726,9 @@ func (a *testAcquiree) requireCanceled(ctx context.Context) { a.t.Fatal("timed out waiting for AcquireJob") } } + +func jsonify(t *testing.T, m map[string]string) string { + bs, err := json.Marshal(m) + require.NoError(t, err) + return string(bs) +} From 5ec3ed5ac87929e8a951ed1896e4cf3026cb7c50 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 28 Feb 2024 13:25:13 +0000 Subject: [PATCH 2/4] chore(docs): update provisioner tag documentation with suggestions from #12315 --- docs/admin/provisioners.md | 88 +++++++++++++++++++++++--------------- 1 file changed, 53 insertions(+), 35 deletions(-) diff --git a/docs/admin/provisioners.md b/docs/admin/provisioners.md index 2e1a0e16b58e3..c9f160341f97d 100644 --- a/docs/admin/provisioners.md +++ b/docs/admin/provisioners.md @@ -47,13 +47,12 @@ the [Helm example](#example-running-an-external-provisioner-with-helm) below. ## Types of provisioners -> Provisioners have two important tags: `scope` and `owner`. Coder sets these -> tags automatically. +Provisioners can broadly be categorized by scope: `organization` or `user`. ### Organization-Scoped Provisioners **Organization-scoped Provisioners** can pick up build jobs created by any user. -These provisioners always have tags `scope=organization owner=""`. +These provisioners always have the implicit tags `scope=organization owner=""`. ```shell coder provisionerd start @@ -62,10 +61,9 @@ coder provisionerd start ### User-scoped Provisioners **User-scoped Provisioners** can only pick up build jobs created from -user-tagged templates. User-scoped provisioners always have tags -`scope=owner owner=`. Unlike the other provisioner types, any Coder user -can run user provisioners, but they have no impact unless there is at least one -template with the `scope=user` provisioner tag. +user-tagged templates. Unlike the other provisioner types, any Coder user can +run user provisioners, but they have no impact unless there is at least one +other template with the `scope=user` provisioner tag. ```shell coder provisionerd start \ @@ -80,59 +78,79 @@ coder templates push on-prem \ ### Provisioner Tags You can use **provisioner tags** to control which provisioners can pick up build -jobs from templates (and corresponding workspaces) with matching tags. +jobs from templates (and corresponding workspaces) with matching explicit tags. + +Provisioners have two implicit tags: `scope` and `owner`. Coder sets these tags +automatically. + +- Organization-scoped provisioners always have the implicit tags + `scope=organization owner=""` +- User-scoped provisioners always have the implicit tags + `scope=owner owner=` + +For example: ```shell +# Start a provisioner with the explicit tags +# environment=on_prem and datacenter=chicago coder provisionerd start \ --tag environment=on_prem \ - --tag data_center=chicago + --tag datacenter=chicago # In another terminal, create/push -# a template that requires this provisioner +# a template that requires the explicit +# tag environment=on_prem coder templates push on-prem \ --provisioner-tag environment=on_prem -# Or, match the provisioner exactly +# Or, match the provisioner's explicit tags exactly coder templates push on-prem-chicago \ --provisioner-tag environment=on_prem \ - --provisioner-tag data_center=chicago + --provisioner-tag datacenter=chicago ``` A provisioner can run a given build job if one of the below is true: -1. The provisioner and job tags are both organization-scoped and both have no - additional tags set, -1. The set of tags of the build job is a subset of the set of tags of the - provisioner. - -This is illustrated in the below table: - -| Provisioner Tags | Job Tags | Can run job? | -| ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------ | -| `{"owner":"","scope":"organization"}` | `{"owner":"","scope":"organization"}` | true | -| `{"owner":"","scope":"organization"}` | `{"environment":"on_prem","owner":"","scope":"organization"}` | false | -| `{"environment":"on_prem","owner":"","scope":"organization"}` | `{"owner":"","scope":"organization"}` | false | -| `{"environment":"on_prem","owner":"","scope":"organization"}` | `{"foo":"bar","owner":"","scope":"organization"}` | true | -| `{"environment":"on_prem","owner":"","scope":"organization"}` | `{"data_center":"chicago","foo":"bar","owner":"","scope":"organization"}` | false | -| `{"data_center":"chicago","environment":"on_prem","owner":"","scope":"organization"}` | `{"foo":"bar","owner":"","scope":"organization"}` | true | -| `{"data_center":"chicago","environment":"on_prem","owner":"","scope":"organization"}` | `{"data_center":"chicago","foo":"bar","owner":"","scope":"organization"}` | true | -| `{"owner":"aaa","scope":"owner"}` | `{"owner":"","scope":"organization"}` | false | -| `{"owner":"aaa","scope":"owner"}` | `{"owner":"aaa","scope":"owner"}` | true | -| `{"owner":"aaa","scope":"owner"}` | `{"owner":"bbb","scope":"owner"}` | false | -| `{"owner":"","scope":"organization"}` | `{"owner":"aaa","scope":"owner"}` | false | +1. A job with no explicit tags can only be run on a provisioner with no explicit + tags. This way you can introduce tagging into your deployment without + disrupting existing provisioners and jobs. +1. If a job has any explicit tags, it can only run on a provisioner with those + explicit tags (the provisioner could have additional tags). The external provisioner in the above example can run build jobs with tags: - `environment=on_prem` -- `data_center=chicago` +- `datacenter=chicago` - `environment=on_prem datacenter=chicago` -- `environment=cloud datacenter=chicago` -- `environment=on_prem datacenter=new_york` However, it will not pick up any build jobs that do not have either of the `environment` or `datacenter` tags set. It will also not pick up any build jobs from templates with the `user` tag set. +This is illustrated in the below table: + +| Provisioner Tags | Job Tags | Can Run Job? | +| ---------------------------------------------------------------- | ---------------------------------------------------------------- | ------------ | +| scope=organization owner= | scope=organization owner= | ✅ | +| scope=organization owner= | scope=organization owner= environment=on-prem | ❌ | +| scope=organization owner= environment=on-prem | scope=organization owner= | ❌ | +| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem | ✅ | +| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem datacenter=chicago | ❌ | +| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem | ✅ | +| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem datacenter=chicago | ✅ | +| scope=owner owner=aaa | scope=organization owner= | ❌ | +| scope=owner owner=aaa | scope=owner owner=aaa | ✅ | +| scope=owner owner=aaa | scope=owner owner=bbb | ❌ | +| scope=organization owner= | scope=owner owner=aaa | ❌ | +| scope=organization owner= | scope=owner owner=aaa environment=on-prem | ❌ | +| scope=owner owner=aaa environment=on-prem | scope=owner owner=aaa | ✅ | +| scope=owner owner=aaa | scope=owner owner=aaa environment=on-prem | ❌ | +| scope=owner owner=aaa environment=on-prem | scope=owner owner=aaa environment=on-prem | ✅ | +| scope=owner owner=aaa environment=on-prem datacenter=chicago | scope=owner owner=aaa environment=on-prem | ✅ | +| scope=owner owner=aaa environment=on-prem | scope=owner owner=aaa environment=on-prem datacenter=chicago | ❌ | +| scope=owner owner=aaa environment=on-prem datacenter=chicago | scope=owner owner=aaa environment=on-prem datacenter=chicago | ✅ | +| scope=owner owner=aaa environment=on-prem datacenter=chicago | scope=owner owner=aaa environment=on-prem datacenter=new_york | ❌ | + ## Example: Running an external provisioner with Helm Coder provides a Helm chart for running external provisioner daemons, which you From 865b73a771b4c71ed26e6b5401f5009b7c7078b4 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Wed, 28 Feb 2024 13:30:23 +0000 Subject: [PATCH 3/4] rm unused func --- coderd/provisionerdserver/acquirer_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/coderd/provisionerdserver/acquirer_test.go b/coderd/provisionerdserver/acquirer_test.go index f6182ae0b4e8e..b6bbe95a981bb 100644 --- a/coderd/provisionerdserver/acquirer_test.go +++ b/coderd/provisionerdserver/acquirer_test.go @@ -726,9 +726,3 @@ func (a *testAcquiree) requireCanceled(ctx context.Context) { a.t.Fatal("timed out waiting for AcquireJob") } } - -func jsonify(t *testing.T, m map[string]string) string { - bs, err := json.Marshal(m) - require.NoError(t, err) - return string(bs) -} From 4f774df7cf0aa52418fd681016f6e52241fad1d9 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Thu, 29 Feb 2024 12:24:15 +0000 Subject: [PATCH 4/4] address PR comments --- coderd/provisionerdserver/acquirer_test.go | 141 ++++++++++----------- docs/admin/provisioners.md | 73 +++++++---- 2 files changed, 117 insertions(+), 97 deletions(-) diff --git a/coderd/provisionerdserver/acquirer_test.go b/coderd/provisionerdserver/acquirer_test.go index b6bbe95a981bb..6d213f082d7b4 100644 --- a/coderd/provisionerdserver/acquirer_test.go +++ b/coderd/provisionerdserver/acquirer_test.go @@ -338,30 +338,12 @@ func TestAcquirer_MatchTags(t *testing.T) { acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: true, }, - { - name: "untagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, - expectAcquire: false, - }, - { - name: "tagged provisioner and untagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, - expectAcquire: false, - }, { name: "tagged provisioner and tagged job", provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: true, }, - { - name: "tagged provisioner and double-tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, - expectAcquire: false, - }, { name: "double-tagged provisioner and tagged job", provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, @@ -375,76 +357,99 @@ func TestAcquirer_MatchTags(t *testing.T) { expectAcquire: true, }, { - name: "owner-scoped provisioner and untagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, - expectAcquire: false, + name: "user-scoped provisioner and user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + expectAcquire: true, }, { - name: "owner-scoped provisioner and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + name: "user-scoped provisioner with tags and user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, expectAcquire: true, }, - { - name: "owner-scoped provisioner and different owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "bbb"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, - expectAcquire: false, + name: "user-scoped provisioner with tags and user-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner with multiple tags and user-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner with multiple tags and user-scoped job with multiple tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, }, { - name: "org-scoped provisioner and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + name: "untagged provisioner and tagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: false, }, { - name: "owner-scoped provisioner and org-scoped job with tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, + name: "tagged provisioner and untagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: false, }, { - name: "owner-scoped provisioner with tags and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, - expectAcquire: true, + name: "tagged provisioner and double-tagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, + expectAcquire: false, }, { - name: "owner-scoped provisioner and owner-scoped job with tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa"}, + name: "double-tagged provisioner and double-tagged job with differing tags", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "new_york"}, expectAcquire: false, }, { - name: "owner-scoped provisioner with tags and owner-scoped job with tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, - expectAcquire: true, + name: "user-scoped provisioner and untagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + expectAcquire: false, }, { - name: "owner-scoped provisioner with multiple tags and owner-scoped job with tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, - expectAcquire: true, + name: "user-scoped provisioner and different user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "bbb"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + expectAcquire: false, }, { - name: "owner-scoped provisioner with tags and owner-scoped job with multiple tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem"}, + name: "org-scoped provisioner and user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: false, }, { - name: "owner-scoped provisioner with tags and owner-scoped job with multiple tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, - expectAcquire: true, + name: "user-scoped provisioner and org-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, + expectAcquire: false, }, { - name: "owner-scoped provisioner with tags and owner-scoped job with differing tags", - provisionerJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "new_york"}, - acquireJobTags: map[string]string{"scope": "owner", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + name: "user-scoped provisioner and user-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + expectAcquire: false, + }, + { + name: "user-scoped provisioner with tags and user-scoped job with multiple tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: false, + }, + { + name: "user-scoped provisioner with tags and user-scoped job with differing tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "new_york"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, expectAcquire: false, }, } @@ -452,7 +457,7 @@ func TestAcquirer_MatchTags(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := testutil.Context(t, testutil.WaitShort/2) + ctx := testutil.Context(t, testutil.WaitShort) // NOTE: explicitly not using fake store for this test. db, ps := dbtestutil.NewDB(t) log := slogtest.Make(t, nil).Leveled(slog.LevelDebug) @@ -503,17 +508,11 @@ func TestAcquirer_MatchTags(t *testing.T) { // turn the JSON map into k=v for readability kvs := func(m map[string]string) string { ss := make([]string, 0, len(m)) - // always start with scope and owner - ss = append(ss, "scope"+"="+m["scope"]) - ss = append(ss, "owner"+"="+m["owner"]) - for k, v := range m { - if k == "scope" || k == "owner" { - continue - } - if len(v) == 0 { - v = `""` + // ensure consistent ordering of tags + for _, k := range []string{"scope", "owner", "environment", "datacenter"} { + if v, found := m[k]; found { + ss = append(ss, k+"="+v) } - ss = append(ss, k+"="+v) } return strings.Join(ss, " ") } diff --git a/docs/admin/provisioners.md b/docs/admin/provisioners.md index c9f160341f97d..e7400ea40166e 100644 --- a/docs/admin/provisioners.md +++ b/docs/admin/provisioners.md @@ -47,7 +47,20 @@ the [Helm example](#example-running-an-external-provisioner-with-helm) below. ## Types of provisioners -Provisioners can broadly be categorized by scope: `organization` or `user`. +Provisioners can broadly be categorized by scope: `organization` or `user`. The +scope of a provisioner can be specified with +[`-tag=scope=`](../cli/provisionerd_start.md#t---tag) when starting the +provisioner daemon. Only users with at least the +[Template Admin](../admin/users.md#roles) role or higher may create +organization-scoped provisioner daemons. + +There are two exceptions: + +- [Built-in provisioners](../cli/server.md#provisioner-daemons) are always + organization-scoped. +- External provisioners started using a + [pre-shared key (PSK)](../cli/provisionerd_start.md#psk) are always + organization-scoped. ### Organization-Scoped Provisioners @@ -62,8 +75,8 @@ coder provisionerd start **User-scoped Provisioners** can only pick up build jobs created from user-tagged templates. Unlike the other provisioner types, any Coder user can -run user provisioners, but they have no impact unless there is at least one -other template with the `scope=user` provisioner tag. +run user provisioners, but they have no impact unless there exists at least one +template with the `scope=user` provisioner tag. ```shell coder provisionerd start \ @@ -86,7 +99,7 @@ automatically. - Organization-scoped provisioners always have the implicit tags `scope=organization owner=""` - User-scoped provisioners always have the implicit tags - `scope=owner owner=` + `scope=user owner=` For example: @@ -125,31 +138,39 @@ The external provisioner in the above example can run build jobs with tags: However, it will not pick up any build jobs that do not have either of the `environment` or `datacenter` tags set. It will also not pick up any build jobs -from templates with the `user` tag set. +from templates with the tag `scope=user` set. This is illustrated in the below table: -| Provisioner Tags | Job Tags | Can Run Job? | -| ---------------------------------------------------------------- | ---------------------------------------------------------------- | ------------ | -| scope=organization owner= | scope=organization owner= | ✅ | -| scope=organization owner= | scope=organization owner= environment=on-prem | ❌ | -| scope=organization owner= environment=on-prem | scope=organization owner= | ❌ | -| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem | ✅ | -| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem datacenter=chicago | ❌ | -| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem | ✅ | -| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem datacenter=chicago | ✅ | -| scope=owner owner=aaa | scope=organization owner= | ❌ | -| scope=owner owner=aaa | scope=owner owner=aaa | ✅ | -| scope=owner owner=aaa | scope=owner owner=bbb | ❌ | -| scope=organization owner= | scope=owner owner=aaa | ❌ | -| scope=organization owner= | scope=owner owner=aaa environment=on-prem | ❌ | -| scope=owner owner=aaa environment=on-prem | scope=owner owner=aaa | ✅ | -| scope=owner owner=aaa | scope=owner owner=aaa environment=on-prem | ❌ | -| scope=owner owner=aaa environment=on-prem | scope=owner owner=aaa environment=on-prem | ✅ | -| scope=owner owner=aaa environment=on-prem datacenter=chicago | scope=owner owner=aaa environment=on-prem | ✅ | -| scope=owner owner=aaa environment=on-prem | scope=owner owner=aaa environment=on-prem datacenter=chicago | ❌ | -| scope=owner owner=aaa environment=on-prem datacenter=chicago | scope=owner owner=aaa environment=on-prem datacenter=chicago | ✅ | -| scope=owner owner=aaa environment=on-prem datacenter=chicago | scope=owner owner=aaa environment=on-prem datacenter=new_york | ❌ | +| Provisioner Tags | Job Tags | Can Run Job? | +| ----------------------------------------------------------------- | ---------------------------------------------------------------- | ------------ | +| scope=organization owner= | scope=organization owner= | ✅ | +| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem | ✅ | +| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem | ✅ | +| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem datacenter=chicago | ✅ | +| scope=user owner=aaa | scope=user owner=aaa | ✅ | +| scope=user owner=aaa environment=on-prem | scope=user owner=aaa | ✅ | +| scope=user owner=aaa environment=on-prem | scope=user owner=aaa environment=on-prem | ✅ | +| scope=user owner=aaa environment=on-prem datacenter=chicago | scope=user owner=aaa environment=on-prem | ✅ | +| scope=user owner=aaa environment=on-prem datacenter=chicago | scope=user owner=aaa environment=on-prem datacenter=chicago | ✅ | +| scope=organization owner= | scope=organization owner= environment=on-prem | ❌ | +| scope=organization owner= environment=on-prem | scope=organization owner= | ❌ | +| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem datacenter=chicago | ❌ | +| scope=organization owner= environment=on-prem datacenter=new_york | scope=organization owner= environment=on-prem datacenter=chicago | ❌ | +| scope=user owner=aaa | scope=organization owner= | ❌ | +| scope=user owner=aaa | scope=user owner=bbb | ❌ | +| scope=organization owner= | scope=user owner=aaa | ❌ | +| scope=organization owner= | scope=user owner=aaa environment=on-prem | ❌ | +| scope=user owner=aaa | scope=user owner=aaa environment=on-prem | ❌ | +| scope=user owner=aaa environment=on-prem | scope=user owner=aaa environment=on-prem datacenter=chicago | ❌ | +| scope=user owner=aaa environment=on-prem datacenter=chicago | scope=user owner=aaa environment=on-prem datacenter=new_york | ❌ | + +> **Note to maintainers:** to generate this table, run the following command and +> copy the output: +> +> ``` +> go test -v -count=1 ./coderd/provisionerdserver/ -test.run='^TestAcquirer_MatchTags/GenTable$' +> ``` ## Example: Running an external provisioner with Helm