Skip to content

feat: add YAML support to server #6934

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 42 commits into from
Apr 7, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
0d08b84
big wip
ammario Mar 30, 2023
fce8e5f
SimpleString isomorphism!!!
ammario Mar 30, 2023
2844497
Support scalars
ammario Mar 30, 2023
4960df8
ComplexObjects work!
ammario Mar 30, 2023
2a2c926
golden files work!
ammario Mar 30, 2023
056437f
Fixup YAML types
ammario Mar 30, 2023
c2428ff
give default value in comment
ammario Mar 30, 2023
8b210dc
Merge remote-tracking branch 'origin/main' into yaml
ammario Mar 30, 2023
26d7f21
Merge remote-tracking branch 'origin/main' into yaml
ammario Mar 31, 2023
38fa539
Add values.YAMLConfigPath
ammario Mar 31, 2023
64b167a
Merge remote-tracking branch 'origin/main' into yaml
ammario Mar 31, 2023
7edea99
Add YAML to core clibase parsing
ammario Mar 31, 2023
3ab35c1
Server Test WIP
ammario Mar 31, 2023
38155da
More WIP
ammario Mar 31, 2023
451c149
Merge remote-tracking branch 'origin/main' into yaml
ammario Apr 1, 2023
c72f67c
hmm
ammario Apr 1, 2023
9d61ca7
Cant find a clean way to do this..
ammario Apr 4, 2023
5b61b7b
Mediocre solution
ammario Apr 4, 2023
4923d92
hmm
ammario Apr 4, 2023
b6f982c
Merge remote-tracking branch 'origin/main' into yaml
ammario Apr 4, 2023
781786b
New, better YAML
ammario Apr 6, 2023
b03999a
clibase passes
ammario Apr 6, 2023
d018838
Fix UnknownOptions errors
ammario Apr 6, 2023
fe93d7f
Work on nil normalization
ammario Apr 6, 2023
0af47bb
Server tests pass!
ammario Apr 6, 2023
4a1df77
Merge remote-tracking branch 'origin/main' into yaml
ammario Apr 6, 2023
64255b3
make gen + self review cleanup
ammario Apr 6, 2023
99d6068
Generate docs
ammario Apr 6, 2023
300484b
make gen
ammario Apr 6, 2023
4cf5a21
Add --debug-options
ammario Apr 6, 2023
abc92d9
Normalize golden files
ammario Apr 6, 2023
a958324
fix log path
ammario Apr 7, 2023
8ead9fc
minor fix
ammario Apr 7, 2023
6eb19f6
Fix mutability bug in PrepareAll
ammario Apr 7, 2023
f77460f
Address review comments
ammario Apr 7, 2023
4e63bd5
Merge remote-tracking branch 'origin/main' into yaml
ammario Apr 7, 2023
2a96c70
Fix windows?
ammario Apr 7, 2023
7ae7cad
Small improvements
ammario Apr 7, 2023
09d5a35
Reduce YAML ident to 2
ammario Apr 7, 2023
d6506c6
Log mystery error
ammario Apr 7, 2023
551e55d
fixup! Log mystery error
ammario Apr 7, 2023
c3f3317
ecdsa
ammario Apr 7, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
make gen + self review cleanup
  • Loading branch information
ammario committed Apr 6, 2023
commit 64255b39668ad5ec3ae5662a57186143de1de2f8
16 changes: 6 additions & 10 deletions cli/clibase/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -270,31 +270,27 @@ func (inv *Invocation) run(state *runState) error {
}
}

// Read configs, if any.
// Read YAML configs, if any.
for _, opt := range inv.Command.Options {
path, ok := opt.Value.(*YAMLConfigPath)
if !ok || path.String() == "" {
continue
}

fi, err := os.OpenFile(path.String(), os.O_RDONLY, 0)
byt, err := os.ReadFile(path.String())
if err != nil {
return xerrors.Errorf("opening config file: %w", err)
return xerrors.Errorf("reading yaml: %w", err)
}
//nolint:revive
defer fi.Close()

dec := yaml.NewDecoder(fi)

var n yaml.Node
err = dec.Decode(&n)
err = yaml.Unmarshal(byt, &n)
if err != nil {
return xerrors.Errorf("decoding config: %w", err)
return xerrors.Errorf("decoding yaml: %w", err)
}

err = inv.Command.Options.UnmarshalYAML(&n)
if err != nil {
return xerrors.Errorf("applying config: %w", err)
return xerrors.Errorf("applying yaml: %w", err)
}
}

Expand Down
5 changes: 3 additions & 2 deletions cli/clibase/values.go
Original file line number Diff line number Diff line change
Expand Up @@ -351,8 +351,9 @@ func (s *Struct[T]) MarshalYAML() (interface{}, error) {
}

func (s *Struct[T]) UnmarshalYAML(n *yaml.Node) error {
// HACK: for compatibility with flags, we set the value to nil if the node
// is empty and T is a slice.
// HACK: for compatibility with flags, we use nil slices instead of empty
// slices. In most cases, nil slices and empty slices are treated
// the same, so this behavior may be removed at some point.
if typ := reflect.TypeOf(s.Value); typ.Kind() == reflect.Slice && len(n.Content) == 0 {
reflect.ValueOf(&s.Value).Elem().Set(reflect.Zero(typ))
return nil
Expand Down
41 changes: 23 additions & 18 deletions cli/clibase/yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,8 @@ func (s *OptionSet) MarshalYAML() (any, error) {
return &root, nil
}

// mapYAMLNodes converts n into a map with keys of form "group.subgroup.option"
// and values of the corresponding YAML nodes.
// mapYAMLNodes converts parent into a map with keys of form "group.subgroup.option"
// and values as the corresponding YAML nodes.
func mapYAMLNodes(parent *yaml.Node) (map[string]*yaml.Node, error) {
if parent.Kind != yaml.MappingNode {
return nil, xerrors.Errorf("expected mapping node, got type %v", parent.Kind)
Expand All @@ -161,30 +161,35 @@ func mapYAMLNodes(parent *yaml.Node) (map[string]*yaml.Node, error) {
}
var (
key string
m = make(map[string]*yaml.Node)
m = make(map[string]*yaml.Node, len(parent.Content)/2)
merr error
)
for i, child := range parent.Content {
if i%2 == 0 {
if child.Kind != yaml.ScalarNode {
// We immediately because the rest of the code is bound to fail
// if we don't know to expect a key or a value.
return nil, xerrors.Errorf("expected scalar node for key, got type %v", child.Kind)
}
key = child.Value
continue
}
// Even if we have a mapping node, we don't know if it's a grouped simple
// option a complex option, so we store both "key" and "group.key". Since
// we're storing pointers, the additional memory is of little concern.

// We don't know if this is a grouped simple option or complex option,
// so we store both "key" and "group.key". Since we're storing pointers,
// the additional memory is of little concern.
m[key] = child
if child.Kind == yaml.MappingNode {
sub, err := mapYAMLNodes(child)
if err != nil {
merr = errors.Join(merr, xerrors.Errorf("mapping node %q: %w", key, err))
continue
}
for k, v := range sub {
m[key+"."+k] = v
}
if child.Kind != yaml.MappingNode {
continue
}

sub, err := mapYAMLNodes(child)
if err != nil {
merr = errors.Join(merr, xerrors.Errorf("mapping node %q: %w", key, err))
continue
}
for k, v := range sub {
m[key+"."+k] = v
}
}

Expand All @@ -209,7 +214,7 @@ func (o *Option) setFromYAMLNode(n *yaml.Node) error {
}
return n.Decode(o.Value)
case yaml.MappingNode:
return xerrors.Errorf("mapping node must implement yaml.Unmarshaler")
return xerrors.Errorf("mapping nodes must implement yaml.Unmarshaler")
default:
return xerrors.Errorf("unexpected node kind %v", n.Kind)
}
Expand All @@ -218,8 +223,8 @@ func (o *Option) setFromYAMLNode(n *yaml.Node) error {
// UnmarshalYAML converts the given YAML node into the option set.
// It is isomorphic with ToYAML.
func (s *OptionSet) UnmarshalYAML(rootNode *yaml.Node) error {
// The rootNode will be a DocumentNode if it's read from a file. Currently,
// we don't support multiple YAML documents.
// The rootNode will be a DocumentNode if it's read from a file. We do
// not support multiple documents in a single file.
if rootNode.Kind == yaml.DocumentNode {
if len(rootNode.Content) != 1 {
return xerrors.Errorf("expected one node in document, got %d", len(rootNode.Content))
Expand Down
2 changes: 1 addition & 1 deletion cli/clitest/clitest.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ func StartWithWaiter(t *testing.T, inv *clibase.Invocation) *ErrorWaiter {
// down Postgres.
t.Logf("command %q timed out during test cleanup", inv.Command.FullName())
}
// When or not this failed the test is up to the caller.
// Whether or not this fails the test is left to the caller.
t.Logf("command %q exited with error: %v", inv.Command.FullName(), err)
errCh <- err
}()
Expand Down
7 changes: 5 additions & 2 deletions cli/testdata/coder_server_--help.golden
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,12 @@ Clients include the coder cli, vs code extension, and the web UI.
Config Options
Use a YAML configuration file when your server launch become unwieldy.

--write-config bool, $CODER_WRITE_CONFIG
-c, --config yaml-config-path, $CODER_CONFIG_PATH
Specify a YAML file to load configuration from.

--write-config bool

Write out the current server as YAML to stdout.
Write out the current server config as YAML to stdout.

Introspection / Logging Options
--log-human string, $CODER_LOGGING_HUMAN (default: /dev/stderr)
Expand Down
3 changes: 3 additions & 0 deletions cli/testdata/server-config.yaml.golden
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,9 @@ oidc:
# {"access_type": "offline"}, type: struct[map[string]string])
authURLParams:
access_type: offline
# Ignore the userinfo endpoint and only use the ID token for user information.
# (default: false, type: bool)
ignoreUserInfo: false
# Change the OIDC default 'groups' claim field. By default, will be 'groups' if
# present in the oidc scopes argument. (default: <unset>, type: string)
groupField: ""
Expand Down
6 changes: 0 additions & 6 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 0 additions & 6 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 0 additions & 10 deletions docs/api/general.md
Original file line number Diff line number Diff line change
Expand Up @@ -371,19 +371,9 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
"flag": "string",
"flag_shorthand": "string",
"group": {
"children": [
{
"children": [],
"description": "string",
"name": "string",
"parent": {},
"yaml_name": "string"
}
],
"description": "string",
"name": "string",
"parent": {
"children": [{}],
"description": "string",
"name": "string",
"parent": {},
Expand Down
53 changes: 6 additions & 47 deletions docs/api/schemas.md
Original file line number Diff line number Diff line change
Expand Up @@ -384,19 +384,9 @@

```json
{
"children": [
{
"children": [],
"description": "string",
"name": "string",
"parent": {},
"yaml_name": "string"
}
],
"description": "string",
"name": "string",
"parent": {
"children": [{}],
"description": "string",
"name": "string",
"parent": {},
Expand All @@ -408,13 +398,12 @@

### Properties

| Name | Type | Required | Restrictions | Description |
| ------------- | --------------------------------------- | -------- | ------------ | ----------- |
| `children` | array of [clibase.Group](#clibasegroup) | false | | |
| `description` | string | false | | |
| `name` | string | false | | |
| `parent` | [clibase.Group](#clibasegroup) | false | | |
| `yaml_name` | string | false | | |
| Name | Type | Required | Restrictions | Description |
| ------------- | ------------------------------ | -------- | ------------ | ----------- |
| `description` | string | false | | |
| `name` | string | false | | |
| `parent` | [clibase.Group](#clibasegroup) | false | | |
| `yaml_name` | string | false | | |

## clibase.HostPort

Expand Down Expand Up @@ -446,19 +435,9 @@
"flag": "string",
"flag_shorthand": "string",
"group": {
"children": [
{
"children": [],
"description": "string",
"name": "string",
"parent": {},
"yaml_name": "string"
}
],
"description": "string",
"name": "string",
"parent": {
"children": [{}],
"description": "string",
"name": "string",
"parent": {},
Expand All @@ -480,19 +459,9 @@
"flag": "string",
"flag_shorthand": "string",
"group": {
"children": [
{
"children": [],
"description": "string",
"name": "string",
"parent": {},
"yaml_name": "string"
}
],
"description": "string",
"name": "string",
"parent": {
"children": [{}],
"description": "string",
"name": "string",
"parent": {},
Expand Down Expand Up @@ -2028,19 +1997,9 @@ CreateParameterRequest is a structure used to create a new parameter value for a
"flag": "string",
"flag_shorthand": "string",
"group": {
"children": [
{
"children": [],
"description": "string",
"name": "string",
"parent": {},
"yaml_name": "string"
}
],
"description": "string",
"name": "string",
"parent": {
"children": [{}],
"description": "string",
"name": "string",
"parent": {},
Expand Down
17 changes: 17 additions & 0 deletions docs/cli/server.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,15 @@ Whether Coder only allows connections to workspaces via the browser.

The directory to cache temporary files. If unspecified and $CACHE_DIRECTORY is set, it will be used for compatibility with systemd.

### -c, --config

| | |
| ----------- | ------------------------------- |
| Type | <code>yaml-config-path</code> |
| Environment | <code>$CODER_CONFIG_PATH</code> |

Specify a YAML file to load configuration from.

### --dangerous-allow-path-app-sharing

| | |
Expand Down Expand Up @@ -790,3 +799,11 @@ Output debug-level logs.
| Environment | <code>$CODER_WILDCARD_ACCESS_URL</code> |

Specifies the wildcard hostname to use for workspace applications in the form "\*.example.com".

### --write-config

| | |
| ---- | ----------------- |
| Type | <code>bool</code> |

<br/>Write out the current server config as YAML to stdout.