Skip to content

Commit a1025f9

Browse files
authored
refactor(examples): pre-parse frontmatter via scripts/examplegen (#9514)
* refactor(examples): pre-parse frontmatter via scripts/examplegen This removes 2 MB from the slim binary. Ref: #9380
1 parent 6fc1f52 commit a1025f9

File tree

9 files changed

+373
-94
lines changed

9 files changed

+373
-94
lines changed

.prettierignore

+3
Original file line numberDiff line numberDiff line change
@@ -83,3 +83,6 @@ scripts/apitypings/testdata/**/*.ts
8383
site/e2e/provisionerGenerated.ts
8484

8585
**/pnpm-lock.yaml
86+
87+
# Ignore generated JSON (e.g. examples/examples.gen.json).
88+
**/*.gen.json

.prettierignore.include

+3
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,6 @@ scripts/apitypings/testdata/**/*.ts
1313
site/e2e/provisionerGenerated.ts
1414

1515
**/pnpm-lock.yaml
16+
17+
# Ignore generated JSON (e.g. examples/examples.gen.json).
18+
**/*.gen.json

Makefile

+9-3
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,8 @@ gen: \
471471
site/.prettierrc.yaml \
472472
site/.prettierignore \
473473
site/.eslintignore \
474-
site/e2e/provisionerGenerated.ts
474+
site/e2e/provisionerGenerated.ts \
475+
examples/examples.gen.json
475476
.PHONY: gen
476477

477478
# Mark all generated files as fresh so make thinks they're up-to-date. This is
@@ -494,6 +495,7 @@ gen/mark-fresh:
494495
site/.prettierignore \
495496
site/.eslintignore \
496497
site/e2e/provisionerGenerated.ts \
498+
examples/examples.gen.json \
497499
"
498500
for file in $$files; do
499501
echo "$$file"
@@ -545,14 +547,18 @@ site/e2e/provisionerGenerated.ts:
545547
../scripts/pnpm_install.sh
546548
pnpm run gen:provisioner
547549

550+
551+
examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates)
552+
go run ./scripts/examplegen/main.go > examples/examples.gen.json
553+
548554
coderd/rbac/object_gen.go: scripts/rbacgen/main.go coderd/rbac/object.go
549555
go run scripts/rbacgen/main.go ./coderd/rbac > coderd/rbac/object_gen.go
550556

551557
docs/admin/prometheus.md: scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics
552558
go run scripts/metricsdocgen/main.go
553559
pnpm run format:write:only ./docs/admin/prometheus.md
554560

555-
docs/cli.md: scripts/clidocgen/main.go $(GO_SRC_FILES)
561+
docs/cli.md: scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES)
556562
BASE_PATH="." go run ./scripts/clidocgen
557563
pnpm run format:write:only ./docs/cli.md ./docs/cli/*.md ./docs/manifest.json
558564

@@ -605,7 +611,7 @@ site/.prettierrc.yaml: .prettierrc.yaml
605611
# - ./ -> ../
606612
# - ./site -> ./
607613
yq \
608-
'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./"))' \
614+
'.overrides[].files |= map(. | sub("^./"; "") | sub("^"; "../") | sub("../site/"; "./") | sub("../!"; "!../"))' \
609615
"$<" >> "$@"
610616

611617
# Combine .gitignore with .prettierignore.include to generate .prettierignore.

examples/examples.gen.json

+149
Large diffs are not rendered by default.

examples/examples.go

+58-90
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,17 @@ package examples
22

33
import (
44
"archive/tar"
5+
"bufio"
56
"bytes"
67
"embed"
8+
"encoding/json"
79
"io"
810
"io/fs"
911
"path"
12+
"sort"
1013
"strings"
1114
"sync"
1215

13-
"github.com/gohugoio/hugo/parser/pageparser"
1416
"golang.org/x/sync/singleflight"
1517
"golang.org/x/xerrors"
1618

@@ -19,6 +21,8 @@ import (
1921

2022
var (
2123
// Only some templates are embedded that we want to display inside the UI.
24+
// The metadata in examples.gen.json is generated via scripts/examplegen.
25+
//go:embed examples.gen.json
2226
//go:embed templates/aws-ecs-container
2327
//go:embed templates/aws-linux
2428
//go:embed templates/aws-windows
@@ -34,112 +38,76 @@ var (
3438
files embed.FS
3539

3640
exampleBasePath = "https://github.com/coder/coder/tree/main/examples/templates/"
37-
examples = make([]codersdk.TemplateExample, 0)
41+
examplesJSON = "examples.gen.json"
42+
parsedExamples []codersdk.TemplateExample
3843
parseExamples sync.Once
39-
archives = singleflight.Group{}
44+
archives singleflight.Group
4045
ErrNotFound = xerrors.New("example not found")
4146
)
4247

4348
const rootDir = "templates"
4449

4550
// List returns all embedded examples.
4651
func List() ([]codersdk.TemplateExample, error) {
47-
var returnError error
52+
var err error
4853
parseExamples.Do(func() {
49-
files, err := fs.Sub(files, rootDir)
50-
if err != nil {
51-
returnError = xerrors.Errorf("get example fs: %w", err)
52-
}
53-
54-
dirs, err := fs.ReadDir(files, ".")
55-
if err != nil {
56-
returnError = xerrors.Errorf("read dir: %w", err)
57-
return
58-
}
59-
60-
for _, dir := range dirs {
61-
if !dir.IsDir() {
62-
continue
63-
}
64-
exampleID := dir.Name()
65-
exampleURL := exampleBasePath + exampleID
66-
// Each one of these is a example!
67-
readme, err := fs.ReadFile(files, path.Join(dir.Name(), "README.md"))
68-
if err != nil {
69-
returnError = xerrors.Errorf("example %q does not contain README.md", exampleID)
70-
return
71-
}
54+
parsedExamples, err = parseAndVerifyExamples()
55+
})
56+
return parsedExamples, err
57+
}
7258

73-
frontMatter, err := pageparser.ParseFrontMatterAndContent(bytes.NewReader(readme))
74-
if err != nil {
75-
returnError = xerrors.Errorf("parse example %q front matter: %w", exampleID, err)
76-
return
77-
}
59+
func parseAndVerifyExamples() (examples []codersdk.TemplateExample, err error) {
60+
f, err := files.Open(examplesJSON)
61+
if err != nil {
62+
return nil, xerrors.Errorf("open %s: %w", examplesJSON, err)
63+
}
64+
defer f.Close()
7865

79-
nameRaw, exists := frontMatter.FrontMatter["name"]
80-
if !exists {
81-
returnError = xerrors.Errorf("example %q front matter does not contain name", exampleID)
82-
return
83-
}
66+
b := bufio.NewReader(f)
8467

85-
name, valid := nameRaw.(string)
86-
if !valid {
87-
returnError = xerrors.Errorf("example %q name isn't a string", exampleID)
88-
return
89-
}
68+
// Discard the first line (code generated by-comment).
69+
_, _, err = b.ReadLine()
70+
if err != nil {
71+
return nil, xerrors.Errorf("read %s: %w", examplesJSON, err)
72+
}
9073

91-
descriptionRaw, exists := frontMatter.FrontMatter["description"]
92-
if !exists {
93-
returnError = xerrors.Errorf("example %q front matter does not contain name", exampleID)
94-
return
95-
}
74+
err = json.NewDecoder(b).Decode(&examples)
75+
if err != nil {
76+
return nil, xerrors.Errorf("decode %s: %w", examplesJSON, err)
77+
}
9678

97-
description, valid := descriptionRaw.(string)
98-
if !valid {
99-
returnError = xerrors.Errorf("example %q description isn't a string", exampleID)
100-
return
101-
}
79+
// Sanity-check: Verify that the examples in the JSON file match the
80+
// embedded files.
81+
var wantEmbedFiles []string
82+
for i, example := range examples {
83+
examples[i].URL = exampleBasePath + example.ID
84+
wantEmbedFiles = append(wantEmbedFiles, example.ID)
85+
}
10286

103-
tags := []string{}
104-
tagsRaw, exists := frontMatter.FrontMatter["tags"]
105-
if exists {
106-
tagsI, valid := tagsRaw.([]interface{})
107-
if !valid {
108-
returnError = xerrors.Errorf("example %q tags isn't a slice: type %T", exampleID, tagsRaw)
109-
return
110-
}
111-
for _, tagI := range tagsI {
112-
tag, valid := tagI.(string)
113-
if !valid {
114-
returnError = xerrors.Errorf("example %q tag isn't a string: type %T", exampleID, tagI)
115-
return
116-
}
117-
tags = append(tags, tag)
118-
}
119-
}
87+
files, err := fs.Sub(files, rootDir)
88+
if err != nil {
89+
return nil, xerrors.Errorf("get templates fs: %w", err)
90+
}
91+
dirs, err := fs.ReadDir(files, ".")
92+
if err != nil {
93+
return nil, xerrors.Errorf("read templates dir: %w", err)
94+
}
95+
var gotEmbedFiles []string
96+
for _, dir := range dirs {
97+
if dir.IsDir() {
98+
gotEmbedFiles = append(gotEmbedFiles, dir.Name())
99+
}
100+
}
120101

121-
var icon string
122-
iconRaw, exists := frontMatter.FrontMatter["icon"]
123-
if exists {
124-
icon, valid = iconRaw.(string)
125-
if !valid {
126-
returnError = xerrors.Errorf("example %q icon isn't a string", exampleID)
127-
return
128-
}
129-
}
102+
sort.Strings(wantEmbedFiles)
103+
sort.Strings(gotEmbedFiles)
104+
want := strings.Join(wantEmbedFiles, ", ")
105+
got := strings.Join(gotEmbedFiles, ", ")
106+
if want != got {
107+
return nil, xerrors.Errorf("mismatch between %s and embedded files: want %q, got %q", examplesJSON, want, got)
108+
}
130109

131-
examples = append(examples, codersdk.TemplateExample{
132-
ID: exampleID,
133-
URL: exampleURL,
134-
Name: name,
135-
Description: description,
136-
Icon: icon,
137-
Tags: tags,
138-
Markdown: string(frontMatter.Content),
139-
})
140-
}
141-
})
142-
return examples, returnError
110+
return examples, nil
143111
}
144112

145113
// Archive returns a tar by example ID.

examples/examples_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
func TestTemplate(t *testing.T) {
1717
t.Parallel()
1818
list, err := examples.List()
19-
require.NoError(t, err)
19+
require.NoError(t, err, "error listing examples, run \"make gen\" to ensure examples are up to date")
2020
require.NotEmpty(t, list)
2121
for _, eg := range list {
2222
eg := eg

0 commit comments

Comments
 (0)